diff --git a/.github/workflows/security-test-workflow.yml b/.github/workflows/security-test-workflow.yml
index 127962210..a096f26a0 100644
--- a/.github/workflows/security-test-workflow.yml
+++ b/.github/workflows/security-test-workflow.yml
@@ -12,7 +12,7 @@ jobs:
   build:
     strategy:
       matrix:
-        java: [ 11, 17 ]
+        java: [ 11, 17, 21 ]
     # Job name
     name: Build and test Alerting
     # This job runs on Linux
@@ -73,20 +73,20 @@ jobs:
         if: env.imagePresent == 'true'
         run: |
           cd ..
-          docker run -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" opensearch-alerting:test
+          docker run -p 9200:9200 -d -p 9600:9600 -e "OPENSEARCH_INITIAL_ADMIN_PASSWORD=myStrongPassword123!" -e "discovery.type=single-node" opensearch-alerting:test
           sleep 120
 
       - name: Run Alerting Test for security enabled test cases
         if: env.imagePresent == 'true'
         run: |
-          cluster_running=`curl -XGET https://localhost:9200/_cat/plugins -u admin:admin --insecure`
+          cluster_running=`curl -XGET https://localhost:9200/_cat/plugins -u admin:myStrongPassword123! --insecure`
           echo $cluster_running
-          security=`curl -XGET https://localhost:9200/_cat/plugins -u admin:admin --insecure |grep opensearch-security|wc -l`
+          security=`curl -XGET https://localhost:9200/_cat/plugins -u admin:myStrongPassword123! --insecure |grep opensearch-security|wc -l`
           echo $security
           if [ $security -gt 0 ]
           then
             echo "Security plugin is available"
-            ./gradlew :alerting:integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=docker-cluster -Dsecurity=true -Dhttps=true -Duser=admin -Dpassword=admin
+            ./gradlew :alerting:integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=docker-cluster -Dsecurity=true -Dhttps=true -Duser=admin -Dpassword=myStrongPassword123!
           else
             echo "Security plugin is NOT available skipping this run as tests without security have already been run"
           fi
diff --git a/.github/workflows/test-workflow.yml b/.github/workflows/test-workflow.yml
index 2ef1196b9..ebc4c2cec 100644
--- a/.github/workflows/test-workflow.yml
+++ b/.github/workflows/test-workflow.yml
@@ -21,7 +21,7 @@ jobs:
       WORKING_DIR: ${{ matrix.working_directory }}.
     strategy:
       matrix:
-        java: [11, 17]
+        java: [11, 17, 21]
     # Job name
     name: Build Alerting with JDK ${{ matrix.java }} on Linux
     # This job runs on Linux
@@ -69,7 +69,7 @@ jobs:
       WORKING_DIR: ${{ matrix.working_directory }}.
     strategy:
       matrix:
-        java: [11, 17]
+        java: [11, 17, 21]
         os: [ windows-latest, macos-latest ]
         include:
           - os: windows-latest
diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md
index ec1a3404b..5342b19d8 100644
--- a/DEVELOPER_GUIDE.md
+++ b/DEVELOPER_GUIDE.md
@@ -76,9 +76,9 @@ When launching a cluster using one of the above commands, logs are placed in `al
 
 1. Setup a local opensearch cluster with security plugin.
 
-   - `./gradlew :alerting:integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=opensearch -Dhttps=true -Dsecurity=true -Duser=admin -Dpassword=admin`
+   - `./gradlew :alerting:integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=opensearch -Dhttps=true -Dsecurity=true -Duser=admin -Dpassword=<admin-password>`
 
-   - `./gradlew :alerting:integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=opensearch -Dhttps=true -Dsecurity=true -Duser=admin -Dpassword=admin --tests "org.opensearch.alerting.MonitorRunnerIT.test execute monitor returns search result"`
+   - `./gradlew :alerting:integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=opensearch -Dhttps=true -Dsecurity=true -Duser=admin -Dpassword=<admin-password> --tests "org.opensearch.alerting.MonitorRunnerIT.test execute monitor returns search result"`
 
 
 #### Building from the IDE
diff --git a/alerting/build.gradle b/alerting/build.gradle
index 9cb2488be..d0f240193 100644
--- a/alerting/build.gradle
+++ b/alerting/build.gradle
@@ -108,7 +108,7 @@ dependencies {
     implementation "org.jetbrains:annotations:13.0"
 
     api project(":alerting-core")
-    implementation "com.github.seancfoley:ipaddress:5.3.3"
+    implementation "com.github.seancfoley:ipaddress:5.4.1"
 
     testImplementation "org.antlr:antlr4-runtime:${versions.antlr4}"
     testImplementation "org.jetbrains.kotlin:kotlin-test:${kotlin_version}"
diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertService.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertService.kt
index 05e35c1b7..e0dc4625e 100644
--- a/alerting/src/main/kotlin/org/opensearch/alerting/AlertService.kt
+++ b/alerting/src/main/kotlin/org/opensearch/alerting/AlertService.kt
@@ -20,6 +20,7 @@ import org.opensearch.action.support.WriteRequest
 import org.opensearch.alerting.alerts.AlertIndices
 import org.opensearch.alerting.model.ActionRunResult
 import org.opensearch.alerting.model.ChainedAlertTriggerRunResult
+import org.opensearch.alerting.model.ClusterMetricsTriggerRunResult
 import org.opensearch.alerting.model.QueryLevelTriggerRunResult
 import org.opensearch.alerting.opensearchapi.firstFailureOrNull
 import org.opensearch.alerting.opensearchapi.retry
@@ -190,6 +191,19 @@ class AlertService(
             )
         }
 
+        // Including a list of triggered clusters for cluster metrics monitors
+        var triggeredClusters: MutableList<String>? = null
+        if (result is ClusterMetricsTriggerRunResult)
+            result.clusterTriggerResults.forEach {
+                if (it.triggered) {
+                    // Add an empty list if one isn't already present
+                    if (triggeredClusters.isNullOrEmpty()) triggeredClusters = mutableListOf()
+
+                    // Add the cluster to the list of triggered clusters
+                    triggeredClusters!!.add(it.cluster)
+                }
+            }
+
         // Merge the alert's error message to the current alert's history
         val updatedHistory = currentAlert?.errorHistory.update(alertError)
         return if (alertError == null && !result.triggered) {
@@ -199,7 +213,8 @@ class AlertService(
                 errorMessage = null,
                 errorHistory = updatedHistory,
                 actionExecutionResults = updatedActionExecutionResults,
-                schemaVersion = IndexUtils.alertIndexSchemaVersion
+                schemaVersion = IndexUtils.alertIndexSchemaVersion,
+                clusters = triggeredClusters
             )
         } else if (alertError == null && currentAlert?.isAcknowledged() == true) {
             null
@@ -212,6 +227,7 @@ class AlertService(
                 errorHistory = updatedHistory,
                 actionExecutionResults = updatedActionExecutionResults,
                 schemaVersion = IndexUtils.alertIndexSchemaVersion,
+                clusters = triggeredClusters
             )
         } else {
             val alertState = if (workflorwRunContext?.auditDelegateMonitorAlerts == true) {
@@ -223,7 +239,8 @@ class AlertService(
                 lastNotificationTime = currentTime, state = alertState, errorMessage = alertError?.message,
                 errorHistory = updatedHistory, actionExecutionResults = updatedActionExecutionResults,
                 schemaVersion = IndexUtils.alertIndexSchemaVersion, executionId = executionId,
-                workflowId = workflorwRunContext?.workflowId ?: ""
+                workflowId = workflorwRunContext?.workflowId ?: "",
+                clusters = triggeredClusters
             )
         }
     }
diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt
index 7c61fe297..e2cb704e7 100644
--- a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt
+++ b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt
@@ -11,15 +11,15 @@ import org.opensearch.alerting.action.ExecuteWorkflowAction
 import org.opensearch.alerting.action.GetDestinationsAction
 import org.opensearch.alerting.action.GetEmailAccountAction
 import org.opensearch.alerting.action.GetEmailGroupAction
-import org.opensearch.alerting.action.GetMonitorAction
+import org.opensearch.alerting.action.GetRemoteIndexesAction
 import org.opensearch.alerting.action.SearchEmailAccountAction
 import org.opensearch.alerting.action.SearchEmailGroupAction
-import org.opensearch.alerting.action.SearchMonitorAction
 import org.opensearch.alerting.alerts.AlertIndices
 import org.opensearch.alerting.core.JobSweeper
 import org.opensearch.alerting.core.ScheduledJobIndices
 import org.opensearch.alerting.core.action.node.ScheduledJobsStatsAction
 import org.opensearch.alerting.core.action.node.ScheduledJobsStatsTransportAction
+import org.opensearch.alerting.core.lock.LockService
 import org.opensearch.alerting.core.resthandler.RestScheduledJobStatsHandler
 import org.opensearch.alerting.core.schedule.JobScheduler
 import org.opensearch.alerting.core.settings.LegacyOpenDistroScheduledJobSettings
@@ -36,6 +36,7 @@ import org.opensearch.alerting.resthandler.RestGetEmailAccountAction
 import org.opensearch.alerting.resthandler.RestGetEmailGroupAction
 import org.opensearch.alerting.resthandler.RestGetFindingsAction
 import org.opensearch.alerting.resthandler.RestGetMonitorAction
+import org.opensearch.alerting.resthandler.RestGetRemoteIndexesAction
 import org.opensearch.alerting.resthandler.RestGetWorkflowAction
 import org.opensearch.alerting.resthandler.RestGetWorkflowAlertsAction
 import org.opensearch.alerting.resthandler.RestIndexMonitorAction
@@ -46,6 +47,7 @@ import org.opensearch.alerting.resthandler.RestSearchMonitorAction
 import org.opensearch.alerting.script.TriggerScript
 import org.opensearch.alerting.service.DeleteMonitorService
 import org.opensearch.alerting.settings.AlertingSettings
+import org.opensearch.alerting.settings.AlertingSettings.Companion.DOC_LEVEL_MONITOR_SHARD_FETCH_SIZE
 import org.opensearch.alerting.settings.DestinationSettings
 import org.opensearch.alerting.settings.LegacyOpenDistroAlertingSettings
 import org.opensearch.alerting.settings.LegacyOpenDistroDestinationSettings
@@ -61,6 +63,7 @@ import org.opensearch.alerting.transport.TransportGetEmailAccountAction
 import org.opensearch.alerting.transport.TransportGetEmailGroupAction
 import org.opensearch.alerting.transport.TransportGetFindingsSearchAction
 import org.opensearch.alerting.transport.TransportGetMonitorAction
+import org.opensearch.alerting.transport.TransportGetRemoteIndexesAction
 import org.opensearch.alerting.transport.TransportGetWorkflowAction
 import org.opensearch.alerting.transport.TransportGetWorkflowAlertsAction
 import org.opensearch.alerting.transport.TransportIndexMonitorAction
@@ -99,6 +102,7 @@ import org.opensearch.core.xcontent.XContentParser
 import org.opensearch.env.Environment
 import org.opensearch.env.NodeEnvironment
 import org.opensearch.index.IndexModule
+import org.opensearch.monitor.jvm.JvmStats
 import org.opensearch.painless.spi.PainlessExtension
 import org.opensearch.painless.spi.Whitelist
 import org.opensearch.painless.spi.WhitelistLoader
@@ -136,6 +140,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R
 
         @JvmField val MONITOR_BASE_URI = "/_plugins/_alerting/monitors"
         @JvmField val WORKFLOW_BASE_URI = "/_plugins/_alerting/workflows"
+        @JvmField val REMOTE_BASE_URI = "/_plugins/_alerting/remote"
         @JvmField val DESTINATION_BASE_URI = "/_plugins/_alerting/destinations"
 
         @JvmField val LEGACY_OPENDISTRO_MONITOR_BASE_URI = "/_opendistro/_alerting/monitors"
@@ -194,7 +199,8 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R
             RestGetWorkflowAlertsAction(),
             RestGetFindingsAction(),
             RestGetWorkflowAction(),
-            RestDeleteWorkflowAction()
+            RestDeleteWorkflowAction(),
+            RestGetRemoteIndexesAction(),
         )
     }
 
@@ -202,9 +208,9 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R
         return listOf(
             ActionPlugin.ActionHandler(ScheduledJobsStatsAction.INSTANCE, ScheduledJobsStatsTransportAction::class.java),
             ActionPlugin.ActionHandler(AlertingActions.INDEX_MONITOR_ACTION_TYPE, TransportIndexMonitorAction::class.java),
-            ActionPlugin.ActionHandler(GetMonitorAction.INSTANCE, TransportGetMonitorAction::class.java),
+            ActionPlugin.ActionHandler(AlertingActions.GET_MONITOR_ACTION_TYPE, TransportGetMonitorAction::class.java),
             ActionPlugin.ActionHandler(ExecuteMonitorAction.INSTANCE, TransportExecuteMonitorAction::class.java),
-            ActionPlugin.ActionHandler(SearchMonitorAction.INSTANCE, TransportSearchMonitorAction::class.java),
+            ActionPlugin.ActionHandler(AlertingActions.SEARCH_MONITORS_ACTION_TYPE, TransportSearchMonitorAction::class.java),
             ActionPlugin.ActionHandler(AlertingActions.DELETE_MONITOR_ACTION_TYPE, TransportDeleteMonitorAction::class.java),
             ActionPlugin.ActionHandler(AlertingActions.ACKNOWLEDGE_ALERTS_ACTION_TYPE, TransportAcknowledgeAlertAction::class.java),
             ActionPlugin.ActionHandler(
@@ -221,7 +227,8 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R
             ActionPlugin.ActionHandler(AlertingActions.INDEX_WORKFLOW_ACTION_TYPE, TransportIndexWorkflowAction::class.java),
             ActionPlugin.ActionHandler(AlertingActions.GET_WORKFLOW_ACTION_TYPE, TransportGetWorkflowAction::class.java),
             ActionPlugin.ActionHandler(AlertingActions.DELETE_WORKFLOW_ACTION_TYPE, TransportDeleteWorkflowAction::class.java),
-            ActionPlugin.ActionHandler(ExecuteWorkflowAction.INSTANCE, TransportExecuteWorkflowAction::class.java)
+            ActionPlugin.ActionHandler(ExecuteWorkflowAction.INSTANCE, TransportExecuteWorkflowAction::class.java),
+            ActionPlugin.ActionHandler(GetRemoteIndexesAction.INSTANCE, TransportGetRemoteIndexesAction::class.java),
         )
     }
 
@@ -254,6 +261,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R
     ): Collection<Any> {
         // Need to figure out how to use the OpenSearch DI classes rather than handwiring things here.
         val settings = environment.settings()
+        val lockService = LockService(client, clusterService)
         alertIndices = AlertIndices(settings, client, threadPool, clusterService)
         runner = MonitorRunnerService
             .registerClusterService(clusterService)
@@ -268,7 +276,9 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R
             .registerTriggerService(TriggerService(scriptService))
             .registerAlertService(AlertService(client, xContentRegistry, alertIndices))
             .registerDocLevelMonitorQueries(DocLevelMonitorQueries(client, clusterService))
+            .registerJvmStats(JvmStats.jvmStats())
             .registerWorkflowService(WorkflowService(client, xContentRegistry))
+            .registerLockService(lockService)
             .registerConsumers()
             .registerDestinationSettings()
         scheduledJobIndices = ScheduledJobIndices(client.admin(), clusterService)
@@ -293,9 +303,9 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R
             settings
         )
 
-        DeleteMonitorService.initialize(client)
+        DeleteMonitorService.initialize(client, lockService)
 
-        return listOf(sweeper, scheduler, runner, scheduledJobIndices, docLevelMonitorQueries, destinationMigrationCoordinator)
+        return listOf(sweeper, scheduler, runner, scheduledJobIndices, docLevelMonitorQueries, destinationMigrationCoordinator, lockService)
     }
 
     override fun getSettings(): List<Setting<*>> {
@@ -325,6 +335,9 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R
             AlertingSettings.ALERT_HISTORY_MAX_DOCS,
             AlertingSettings.ALERT_HISTORY_RETENTION_PERIOD,
             AlertingSettings.ALERTING_MAX_MONITORS,
+            AlertingSettings.PERCOLATE_QUERY_DOCS_SIZE_MEMORY_PERCENTAGE_LIMIT,
+            DOC_LEVEL_MONITOR_SHARD_FETCH_SIZE,
+            AlertingSettings.PERCOLATE_QUERY_MAX_NUM_DOCS_IN_MEMORY,
             AlertingSettings.REQUEST_TIMEOUT,
             AlertingSettings.MAX_ACTION_THROTTLE_VALUE,
             AlertingSettings.FILTER_BY_BACKEND_ROLES,
@@ -345,6 +358,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R
             LegacyOpenDistroAlertingSettings.REQUEST_TIMEOUT,
             LegacyOpenDistroAlertingSettings.MAX_ACTION_THROTTLE_VALUE,
             LegacyOpenDistroAlertingSettings.FILTER_BY_BACKEND_ROLES,
+            AlertingSettings.DOC_LEVEL_MONITOR_FETCH_ONLY_QUERY_FIELDS_ENABLED,
             DestinationSettings.EMAIL_USERNAME,
             DestinationSettings.EMAIL_PASSWORD,
             DestinationSettings.ALLOW_LIST,
@@ -357,7 +371,9 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R
             AlertingSettings.FINDING_HISTORY_MAX_DOCS,
             AlertingSettings.FINDING_HISTORY_INDEX_MAX_AGE,
             AlertingSettings.FINDING_HISTORY_ROLLOVER_PERIOD,
-            AlertingSettings.FINDING_HISTORY_RETENTION_PERIOD
+            AlertingSettings.FINDING_HISTORY_RETENTION_PERIOD,
+            AlertingSettings.FINDINGS_INDEXING_BATCH_SIZE,
+            AlertingSettings.REMOTE_MONITORING_ENABLED
         )
     }
 
diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/DocumentLevelMonitorRunner.kt b/alerting/src/main/kotlin/org/opensearch/alerting/DocumentLevelMonitorRunner.kt
index f69151b13..ff939418a 100644
--- a/alerting/src/main/kotlin/org/opensearch/alerting/DocumentLevelMonitorRunner.kt
+++ b/alerting/src/main/kotlin/org/opensearch/alerting/DocumentLevelMonitorRunner.kt
@@ -8,14 +8,17 @@ package org.opensearch.alerting
 import org.apache.logging.log4j.LogManager
 import org.opensearch.ExceptionsHelper
 import org.opensearch.OpenSearchStatusException
+import org.opensearch.action.DocWriteRequest
+import org.opensearch.action.admin.indices.refresh.RefreshAction
+import org.opensearch.action.admin.indices.refresh.RefreshRequest
+import org.opensearch.action.bulk.BulkRequest
+import org.opensearch.action.bulk.BulkResponse
 import org.opensearch.action.index.IndexRequest
-import org.opensearch.action.index.IndexResponse
 import org.opensearch.action.search.SearchAction
 import org.opensearch.action.search.SearchRequest
 import org.opensearch.action.search.SearchResponse
-import org.opensearch.action.support.WriteRequest
-import org.opensearch.alerting.model.DocumentExecutionContext
 import org.opensearch.alerting.model.DocumentLevelTriggerRunResult
+import org.opensearch.alerting.model.IndexExecutionContext
 import org.opensearch.alerting.model.InputRunResults
 import org.opensearch.alerting.model.MonitorMetadata
 import org.opensearch.alerting.model.MonitorRunResult
@@ -27,7 +30,6 @@ import org.opensearch.alerting.util.IndexUtils
 import org.opensearch.alerting.util.defaultToPerExecutionAction
 import org.opensearch.alerting.util.getActionExecutionPolicy
 import org.opensearch.alerting.workflow.WorkflowRunContext
-import org.opensearch.client.Client
 import org.opensearch.client.node.NodeClient
 import org.opensearch.cluster.metadata.IndexMetadata
 import org.opensearch.cluster.routing.Preference
@@ -56,17 +58,30 @@ import org.opensearch.index.IndexNotFoundException
 import org.opensearch.index.query.BoolQueryBuilder
 import org.opensearch.index.query.Operator
 import org.opensearch.index.query.QueryBuilders
+import org.opensearch.index.seqno.SequenceNumbers
+import org.opensearch.indices.IndexClosedException
 import org.opensearch.percolator.PercolateQueryBuilderExt
+import org.opensearch.search.SearchHit
 import org.opensearch.search.SearchHits
 import org.opensearch.search.builder.SearchSourceBuilder
 import org.opensearch.search.sort.SortOrder
 import java.io.IOException
 import java.time.Instant
 import java.util.UUID
+import java.util.stream.Collectors
 import kotlin.math.max
 
-object DocumentLevelMonitorRunner : MonitorRunner() {
+class DocumentLevelMonitorRunner : MonitorRunner() {
     private val logger = LogManager.getLogger(javaClass)
+    var nonPercolateSearchesTimeTakenStat = 0L
+    var percolateQueriesTimeTakenStat = 0L
+    var totalDocsQueriedStat = 0L
+    var docTransformTimeTakenStat = 0L
+    var totalDocsSizeInBytesStat = 0L
+    var docsSizeOfBatchInBytes = 0L
+    /* Contains list of docs source that are held in memory to submit to percolate query against query index.
+            * Docs are fetched from the source index per shard and transformed.*/
+    val transformedDocs = mutableListOf<Pair<String, TransformedDocDto>>()
 
     override suspend fun runMonitor(
         monitor: Monitor,
@@ -120,12 +135,12 @@ object DocumentLevelMonitorRunner : MonitorRunner() {
 
         try {
             // Resolve all passed indices to concrete indices
-            val concreteIndices = IndexUtils.resolveAllIndices(
+            val allConcreteIndices = IndexUtils.resolveAllIndices(
                 docLevelMonitorInput.indices,
                 monitorCtx.clusterService!!,
                 monitorCtx.indexNameExpressionResolver!!
             )
-            if (concreteIndices.isEmpty()) {
+            if (allConcreteIndices.isEmpty()) {
                 logger.error("indices not found-${docLevelMonitorInput.indices.joinToString(",")}")
                 throw IndexNotFoundException(docLevelMonitorInput.indices.joinToString(","))
             }
@@ -141,7 +156,7 @@ object DocumentLevelMonitorRunner : MonitorRunner() {
             // cleanup old indices that are not monitored anymore from the same monitor
             val runContextKeys = updatedLastRunContext.keys.toMutableSet()
             for (ind in runContextKeys) {
-                if (!concreteIndices.contains(ind)) {
+                if (!allConcreteIndices.contains(ind)) {
                     updatedLastRunContext.remove(ind)
                 }
             }
@@ -149,13 +164,32 @@ object DocumentLevelMonitorRunner : MonitorRunner() {
             // Map of document ids per index when monitor is workflow delegate and has chained findings
             val matchingDocIdsPerIndex = workflowRunContext?.matchingDocIdsPerIndex
 
+            val concreteIndicesSeenSoFar = mutableListOf<String>()
+            val updatedIndexNames = mutableListOf<String>()
             docLevelMonitorInput.indices.forEach { indexName ->
-                val concreteIndices = IndexUtils.resolveAllIndices(
+                var concreteIndices = IndexUtils.resolveAllIndices(
                     listOf(indexName),
                     monitorCtx.clusterService!!,
                     monitorCtx.indexNameExpressionResolver!!
                 )
+                var lastWriteIndex: String? = null
+                if (IndexUtils.isAlias(indexName, monitorCtx.clusterService!!.state()) ||
+                    IndexUtils.isDataStream(indexName, monitorCtx.clusterService!!.state())
+                ) {
+                    lastWriteIndex = concreteIndices.find { lastRunContext.containsKey(it) }
+                    if (lastWriteIndex != null) {
+                        val lastWriteIndexCreationDate =
+                            IndexUtils.getCreationDateForIndex(lastWriteIndex, monitorCtx.clusterService!!.state())
+                        concreteIndices = IndexUtils.getNewestIndicesByCreationDate(
+                            concreteIndices,
+                            monitorCtx.clusterService!!.state(),
+                            lastWriteIndexCreationDate
+                        )
+                    }
+                }
+                concreteIndicesSeenSoFar.addAll(concreteIndices)
                 val updatedIndexName = indexName.replace("*", "_")
+                updatedIndexNames.add(updatedIndexName)
                 val conflictingFields = monitorCtx.docLevelMonitorQueries!!.getAllConflictingFields(
                     monitorCtx.clusterService!!.state(),
                     concreteIndices
@@ -174,12 +208,21 @@ object DocumentLevelMonitorRunner : MonitorRunner() {
                     }
 
                     // Prepare updatedLastRunContext for each index
-                    val indexUpdatedRunContext = updateLastRunContext(
+                    val indexUpdatedRunContext = initializeNewLastRunContext(
                         indexLastRunContext.toMutableMap(),
                         monitorCtx,
-                        concreteIndexName
+                        concreteIndexName,
                     ) as MutableMap<String, Any>
-                    updatedLastRunContext[concreteIndexName] = indexUpdatedRunContext
+                    if (IndexUtils.isAlias(indexName, monitorCtx.clusterService!!.state()) ||
+                        IndexUtils.isDataStream(indexName, monitorCtx.clusterService!!.state())
+                    ) {
+                        if (concreteIndexName == IndexUtils.getWriteIndex(indexName, monitorCtx.clusterService!!.state())) {
+                            updatedLastRunContext.remove(lastWriteIndex)
+                            updatedLastRunContext[concreteIndexName] = indexUpdatedRunContext
+                        }
+                    } else {
+                        updatedLastRunContext[concreteIndexName] = indexUpdatedRunContext
+                    }
 
                     val count: Int = indexLastRunContext["shards_count"] as Int
                     for (i: Int in 0 until count) {
@@ -192,44 +235,65 @@ object DocumentLevelMonitorRunner : MonitorRunner() {
                         }
                     }
 
-                    // Prepare DocumentExecutionContext for each index
-                    val docExecutionContext = DocumentExecutionContext(queries, indexLastRunContext, indexUpdatedRunContext)
-
-                    val matchingDocs = getMatchingDocs(
-                        monitor,
-                        monitorCtx,
-                        docExecutionContext,
+                    val fieldsToBeQueried = mutableSetOf<String>()
+                    if (monitorCtx.fetchOnlyQueryFieldNames) {
+                        for (it in queries) {
+                            if (it.queryFieldNames.isEmpty()) {
+                                fieldsToBeQueried.clear()
+                                logger.debug(
+                                    "Monitor ${monitor.id} : " +
+                                        "Doc Level query ${it.id} : ${it.query} doesn't have queryFieldNames populated. " +
+                                        "Cannot optimize monitor to fetch only query-relevant fields. " +
+                                        "Querying entire doc source."
+                                )
+                                break
+                            }
+                            fieldsToBeQueried.addAll(it.queryFieldNames)
+                        }
+                        if (fieldsToBeQueried.isNotEmpty())
+                            logger.debug(
+                                "Monitor ${monitor.id} Querying only fields " +
+                                    "${fieldsToBeQueried.joinToString()} instead of entire _source of documents"
+                            )
+                    }
+                    val indexExecutionContext = IndexExecutionContext(
+                        queries,
+                        indexLastRunContext,
+                        indexUpdatedRunContext,
                         updatedIndexName,
                         concreteIndexName,
                         conflictingFields.toList(),
-                        matchingDocIdsPerIndex?.get(concreteIndexName)
+                        matchingDocIdsPerIndex?.get(concreteIndexName),
                     )
 
-                    if (matchingDocs.isNotEmpty()) {
-                        val matchedQueriesForDocs = getMatchedQueries(
-                            monitorCtx,
-                            matchingDocs.map { it.second },
-                            monitor,
-                            monitorMetadata,
-                            updatedIndexName,
-                            concreteIndexName
-                        )
-
-                        matchedQueriesForDocs.forEach { hit ->
-                            val id = hit.id
-                                .replace("_${updatedIndexName}_${monitor.id}", "")
-                                .replace("_${concreteIndexName}_${monitor.id}", "")
-
-                            val docIndices = hit.field("_percolator_document_slot").values.map { it.toString().toInt() }
-                            docIndices.forEach { idx ->
-                                val docIndex = "${matchingDocs[idx].first}|$concreteIndexName"
-                                inputRunResults.getOrPut(id) { mutableSetOf() }.add(docIndex)
-                                docsToQueries.getOrPut(docIndex) { mutableListOf() }.add(id)
-                            }
-                        }
+                    fetchShardDataAndMaybeExecutePercolateQueries(
+                        monitor,
+                        monitorCtx,
+                        indexExecutionContext,
+                        monitorMetadata,
+                        inputRunResults,
+                        docsToQueries,
+                        updatedIndexNames,
+                        concreteIndicesSeenSoFar,
+                        ArrayList(fieldsToBeQueried)
+                    ) { shard, maxSeqNo -> // function passed to update last run context with new max sequence number
+                        indexExecutionContext.updatedLastRunContext[shard] = maxSeqNo
                     }
                 }
             }
+            /* if all indices are covered still in-memory docs size limit is not breached we would need to submit
+               the percolate query at the end */
+            if (transformedDocs.isNotEmpty()) {
+                performPercolateQueryAndResetCounters(
+                    monitorCtx,
+                    monitor,
+                    monitorMetadata,
+                    updatedIndexNames,
+                    concreteIndicesSeenSoFar,
+                    inputRunResults,
+                    docsToQueries,
+                )
+            }
             monitorResult = monitorResult.copy(inputResults = InputRunResults(listOf(inputRunResults)))
 
             /*
@@ -249,10 +313,7 @@ object DocumentLevelMonitorRunner : MonitorRunner() {
             // If there are no triggers defined, we still want to generate findings
             if (monitor.triggers.isEmpty()) {
                 if (dryrun == false && monitor.id != Monitor.NO_ID) {
-                    docsToQueries.forEach {
-                        val triggeredQueries = it.value.map { queryId -> idQueryMap[queryId]!! }
-                        createFindings(monitor, monitorCtx, triggeredQueries, it.key, true)
-                    }
+                    createFindings(monitor, monitorCtx, docsToQueries, idQueryMap, true)
                 }
             } else {
                 monitor.triggers.forEach {
@@ -289,6 +350,9 @@ object DocumentLevelMonitorRunner : MonitorRunner() {
                     monitorMetadata.copy(lastRunContext = updatedLastRunContext),
                     true
                 )
+            } else {
+                // Clean up any queries created by the dry run monitor
+                monitorCtx.docLevelMonitorQueries!!.deleteDocLevelQueriesOnDryRun(monitorMetadata)
             }
 
             // TODO: Update the Document as part of the Trigger and return back the trigger action result
@@ -303,6 +367,22 @@ object DocumentLevelMonitorRunner : MonitorRunner() {
                 e
             )
             return monitorResult.copy(error = alertingException, inputResults = InputRunResults(emptyList(), alertingException))
+        } finally {
+            logger.debug(
+                "PERF_DEBUG_STATS: Monitor ${monitor.id} " +
+                    "Time spent on fetching data from shards in millis: $nonPercolateSearchesTimeTakenStat"
+            )
+            logger.debug(
+                "PERF_DEBUG_STATS: Monitor {} Time spent on percolate queries in millis: {}",
+                monitor.id,
+                percolateQueriesTimeTakenStat
+            )
+            logger.debug(
+                "PERF_DEBUG_STATS: Monitor {} Time spent on transforming doc fields in millis: {}",
+                monitor.id,
+                docTransformTimeTakenStat
+            )
+            logger.debug("PERF_DEBUG_STATS: Monitor {} Num docs queried: {}", monitor.id, totalDocsQueriedStat)
         }
     }
 
@@ -341,7 +421,7 @@ object DocumentLevelMonitorRunner : MonitorRunner() {
         trigger: DocumentLevelTrigger,
         monitor: Monitor,
         idQueryMap: Map<String, DocLevelQuery>,
-        docsToQueries: Map<String, List<String>>,
+        docsToQueries: MutableMap<String, MutableList<String>>,
         queryToDocIds: Map<DocLevelQuery, Set<String>>,
         dryrun: Boolean,
         workflowRunContext: WorkflowRunContext?,
@@ -350,35 +430,33 @@ object DocumentLevelMonitorRunner : MonitorRunner() {
         val triggerCtx = DocumentLevelTriggerExecutionContext(monitor, trigger)
         val triggerResult = monitorCtx.triggerService!!.runDocLevelTrigger(monitor, trigger, queryToDocIds)
 
-        val findings = mutableListOf<String>()
-        val findingDocPairs = mutableListOf<Pair<String, String>>()
+        val triggerFindingDocPairs = mutableListOf<Pair<String, String>>()
 
         // TODO: Implement throttling for findings
-        docsToQueries.forEach {
-            val triggeredQueries = it.value.map { queryId -> idQueryMap[queryId]!! }
-            val findingId = createFindings(
-                monitor,
-                monitorCtx,
-                triggeredQueries,
-                it.key,
-                !dryrun && monitor.id != Monitor.NO_ID,
-                executionId
-            )
-            findings.add(findingId)
+        val findingToDocPairs = createFindings(
+            monitor,
+            monitorCtx,
+            docsToQueries,
+            idQueryMap,
+            !dryrun && monitor.id != Monitor.NO_ID,
+            executionId
+        )
 
-            if (triggerResult.triggeredDocs.contains(it.key)) {
-                findingDocPairs.add(Pair(findingId, it.key))
+        findingToDocPairs.forEach {
+            // Only pick those entries whose docs have triggers associated with them
+            if (triggerResult.triggeredDocs.contains(it.second)) {
+                triggerFindingDocPairs.add(Pair(it.first, it.second))
             }
         }
 
         val actionCtx = triggerCtx.copy(
             triggeredDocs = triggerResult.triggeredDocs,
-            relatedFindings = findings,
+            relatedFindings = findingToDocPairs.map { it.first },
             error = monitorResult.error ?: triggerResult.error
         )
 
         val alerts = mutableListOf<Alert>()
-        findingDocPairs.forEach {
+        triggerFindingDocPairs.forEach {
             val alert = monitorCtx.alertService!!.composeDocLevelAlert(
                 listOf(it.first),
                 listOf(it.second),
@@ -437,51 +515,92 @@ object DocumentLevelMonitorRunner : MonitorRunner() {
         return triggerResult
     }
 
+    /**
+     * 1. Bulk index all findings based on shouldCreateFinding flag
+     * 2. invoke publishFinding() to kickstart auto-correlations
+     * 3. Returns a list of pairs for finding id to doc id
+     */
     private suspend fun createFindings(
         monitor: Monitor,
         monitorCtx: MonitorRunnerExecutionContext,
-        docLevelQueries: List<DocLevelQuery>,
-        matchingDocId: String,
+        docsToQueries: MutableMap<String, MutableList<String>>,
+        idQueryMap: Map<String, DocLevelQuery>,
         shouldCreateFinding: Boolean,
         workflowExecutionId: String? = null,
-    ): String {
-        // Before the "|" is the doc id and after the "|" is the index
-        val docIndex = matchingDocId.split("|")
+    ): List<Pair<String, String>> {
 
-        val finding = Finding(
-            id = UUID.randomUUID().toString(),
-            relatedDocIds = listOf(docIndex[0]),
-            correlatedDocIds = listOf(docIndex[0]),
-            monitorId = monitor.id,
-            monitorName = monitor.name,
-            index = docIndex[1],
-            docLevelQueries = docLevelQueries,
-            timestamp = Instant.now(),
-            executionId = workflowExecutionId
-        )
+        val findingDocPairs = mutableListOf<Pair<String, String>>()
+        val findings = mutableListOf<Finding>()
+        val indexRequests = mutableListOf<IndexRequest>()
 
-        val findingStr = finding.toXContent(XContentBuilder.builder(XContentType.JSON.xContent()), ToXContent.EMPTY_PARAMS).string()
-        logger.debug("Findings: $findingStr")
+        docsToQueries.forEach {
+            val triggeredQueries = it.value.map { queryId -> idQueryMap[queryId]!! }
 
-        if (shouldCreateFinding) {
-            val indexRequest = IndexRequest(monitor.dataSources.findingsIndex)
-                .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
-                .source(findingStr, XContentType.JSON)
-                .id(finding.id)
-                .routing(finding.id)
+            // Before the "|" is the doc id and after the "|" is the index
+            val docIndex = it.key.split("|")
 
-            monitorCtx.client!!.suspendUntil<Client, IndexResponse> {
-                monitorCtx.client!!.index(indexRequest, it)
+            val finding = Finding(
+                id = UUID.randomUUID().toString(),
+                relatedDocIds = listOf(docIndex[0]),
+                correlatedDocIds = listOf(docIndex[0]),
+                monitorId = monitor.id,
+                monitorName = monitor.name,
+                index = docIndex[1],
+                docLevelQueries = triggeredQueries,
+                timestamp = Instant.now(),
+                executionId = workflowExecutionId
+            )
+            findingDocPairs.add(Pair(finding.id, it.key))
+            findings.add(finding)
+
+            val findingStr =
+                finding.toXContent(XContentBuilder.builder(XContentType.JSON.xContent()), ToXContent.EMPTY_PARAMS)
+                    .string()
+            logger.debug("Findings: $findingStr")
+
+            if (shouldCreateFinding) {
+                indexRequests += IndexRequest(monitor.dataSources.findingsIndex)
+                    .source(findingStr, XContentType.JSON)
+                    .id(finding.id)
+                    .opType(DocWriteRequest.OpType.CREATE)
             }
         }
 
+        if (indexRequests.isNotEmpty()) {
+            bulkIndexFindings(monitor, monitorCtx, indexRequests)
+        }
+
         try {
-            publishFinding(monitor, monitorCtx, finding)
+            findings.forEach { finding ->
+                publishFinding(monitor, monitorCtx, finding)
+            }
         } catch (e: Exception) {
             // suppress exception
             logger.error("Optional finding callback failed", e)
         }
-        return finding.id
+        return findingDocPairs
+    }
+
+    private suspend fun bulkIndexFindings(
+        monitor: Monitor,
+        monitorCtx: MonitorRunnerExecutionContext,
+        indexRequests: List<IndexRequest>
+    ) {
+        indexRequests.chunked(monitorCtx.findingsIndexBatchSize).forEach { batch ->
+            val bulkResponse: BulkResponse = monitorCtx.client!!.suspendUntil {
+                bulk(BulkRequest().add(batch), it)
+            }
+            if (bulkResponse.hasFailures()) {
+                bulkResponse.items.forEach { item ->
+                    if (item.isFailed) {
+                        logger.error("Failed indexing the finding ${item.id} of monitor [${monitor.id}]")
+                    }
+                }
+            } else {
+                logger.debug("[${bulkResponse.items.size}] All findings successfully indexed.")
+            }
+        }
+        monitorCtx.client!!.execute(RefreshAction.INSTANCE, RefreshRequest(monitor.dataSources.findingsIndex))
     }
 
     private fun publishFinding(
@@ -501,17 +620,16 @@ object DocumentLevelMonitorRunner : MonitorRunner() {
         )
     }
 
-    private suspend fun updateLastRunContext(
+    private fun initializeNewLastRunContext(
         lastRunContext: Map<String, Any>,
         monitorCtx: MonitorRunnerExecutionContext,
-        index: String
+        index: String,
     ): Map<String, Any> {
         val count: Int = getShardsCount(monitorCtx.clusterService!!, index)
         val updatedLastRunContext = lastRunContext.toMutableMap()
         for (i: Int in 0 until count) {
             val shard = i.toString()
-            val maxSeqNo: Long = getMaxSeqNo(monitorCtx.client!!, index, shard)
-            updatedLastRunContext[shard] = maxSeqNo.toString()
+            updatedLastRunContext[shard] = SequenceNumbers.UNASSIGNED_SEQ_NO.toString()
         }
         return updatedLastRunContext
     }
@@ -543,83 +661,166 @@ object DocumentLevelMonitorRunner : MonitorRunner() {
         return indexCreationDate > lastExecutionTime.toEpochMilli()
     }
 
-    /**
-     * Get the current max seq number of the shard. We find it by searching the last document
-     *  in the primary shard.
-     */
-    private suspend fun getMaxSeqNo(client: Client, index: String, shard: String): Long {
-        val request: SearchRequest = SearchRequest()
-            .indices(index)
-            .preference("_shards:$shard")
-            .source(
-                SearchSourceBuilder()
-                    .version(true)
-                    .sort("_seq_no", SortOrder.DESC)
-                    .seqNoAndPrimaryTerm(true)
-                    .query(QueryBuilders.matchAllQuery())
-                    .size(1)
-            )
-        val response: SearchResponse = client.suspendUntil { client.search(request, it) }
-        if (response.status() !== RestStatus.OK) {
-            throw IOException("Failed to get max seq no for shard: $shard")
-        }
-        if (response.hits.hits.isEmpty()) {
-            return -1L
-        }
-
-        return response.hits.hits[0].seqNo
-    }
-
     private fun getShardsCount(clusterService: ClusterService, index: String): Int {
         val allShards: List<ShardRouting> = clusterService!!.state().routingTable().allShards(index)
         return allShards.filter { it.primary() }.size
     }
 
-    private suspend fun getMatchingDocs(
+    /** 1. Fetch data per shard for given index. (only 10000 docs are fetched.
+     * needs to be converted to scroll if not performant enough)
+     *  2. Transform documents to conform to format required for percolate query
+     *  3a. Check if docs in memory are crossing threshold defined by setting.
+     *  3b. If yes, perform percolate query and update docToQueries Map with all hits from percolate queries */
+    private suspend fun fetchShardDataAndMaybeExecutePercolateQueries(
         monitor: Monitor,
         monitorCtx: MonitorRunnerExecutionContext,
-        docExecutionCtx: DocumentExecutionContext,
-        index: String,
-        concreteIndex: String,
-        conflictingFields: List<String>,
-        docIds: List<String>? = null
-    ): List<Pair<String, BytesReference>> {
-        val count: Int = docExecutionCtx.updatedLastRunContext["shards_count"] as Int
-        val matchingDocs = mutableListOf<Pair<String, BytesReference>>()
+        indexExecutionCtx: IndexExecutionContext,
+        monitorMetadata: MonitorMetadata,
+        inputRunResults: MutableMap<String, MutableSet<String>>,
+        docsToQueries: MutableMap<String, MutableList<String>>,
+        monitorInputIndices: List<String>,
+        concreteIndices: List<String>,
+        fieldsToBeQueried: List<String>,
+        updateLastRunContext: (String, String) -> Unit
+    ) {
+        val count: Int = indexExecutionCtx.updatedLastRunContext["shards_count"] as Int
         for (i: Int in 0 until count) {
             val shard = i.toString()
             try {
-                val maxSeqNo: Long = docExecutionCtx.updatedLastRunContext[shard].toString().toLong()
-                val prevSeqNo = docExecutionCtx.lastRunContext[shard].toString().toLongOrNull()
-
-                val hits: SearchHits = searchShard(
+                val prevSeqNo = indexExecutionCtx.lastRunContext[shard].toString().toLongOrNull()
+                val from = prevSeqNo ?: SequenceNumbers.NO_OPS_PERFORMED
+                var to: Long = Long.MAX_VALUE
+                while (to >= from) {
+                    val hits: SearchHits = searchShard(
+                        monitorCtx,
+                        indexExecutionCtx.concreteIndexName,
+                        shard,
+                        from,
+                        to,
+                        indexExecutionCtx.docIds,
+                        fieldsToBeQueried,
+                    )
+                    if (hits.hits.isEmpty()) {
+                        if (to == Long.MAX_VALUE) {
+                            updateLastRunContext(shard, (prevSeqNo ?: SequenceNumbers.NO_OPS_PERFORMED).toString()) // didn't find any docs
+                        }
+                        break
+                    }
+                    if (to == Long.MAX_VALUE) { // max sequence number of shard needs to be computed
+                        updateLastRunContext(shard, hits.hits[0].seqNo.toString())
+                    }
+                    val leastSeqNoFromHits = hits.hits.last().seqNo
+                    to = leastSeqNoFromHits - 1
+                    val startTime = System.currentTimeMillis()
+                    transformedDocs.addAll(
+                        transformSearchHitsAndReconstructDocs(
+                            hits,
+                            indexExecutionCtx.indexName,
+                            indexExecutionCtx.concreteIndexName,
+                            monitor.id,
+                            indexExecutionCtx.conflictingFields,
+                        )
+                    )
+                    if (
+                        transformedDocs.isNotEmpty() &&
+                        shouldPerformPercolateQueryAndFlushInMemoryDocs(transformedDocs.size, monitorCtx)
+                    ) {
+                        performPercolateQueryAndResetCounters(
+                            monitorCtx,
+                            monitor,
+                            monitorMetadata,
+                            monitorInputIndices,
+                            concreteIndices,
+                            inputRunResults,
+                            docsToQueries,
+                        )
+                    }
+                    docTransformTimeTakenStat += System.currentTimeMillis() - startTime
+                }
+            } catch (e: Exception) {
+                logger.error(
+                    "Monitor ${monitor.id} :" +
+                        "Failed to run fetch data from shard [$shard] of index [${indexExecutionCtx.concreteIndexName}]. " +
+                        "Error: ${e.message}",
+                    e
+                )
+                if (e is IndexClosedException) {
+                    throw e
+                }
+            }
+            if (
+                transformedDocs.isNotEmpty() &&
+                shouldPerformPercolateQueryAndFlushInMemoryDocs(transformedDocs.size, monitorCtx)
+            ) {
+                performPercolateQueryAndResetCounters(
                     monitorCtx,
-                    concreteIndex,
-                    shard,
-                    prevSeqNo,
-                    maxSeqNo,
-                    null,
-                    docIds
+                    monitor,
+                    monitorMetadata,
+                    monitorInputIndices,
+                    concreteIndices,
+                    inputRunResults,
+                    docsToQueries,
                 )
+            }
+        }
+    }
 
-                if (hits.hits.isNotEmpty()) {
-                    matchingDocs.addAll(getAllDocs(hits, index, concreteIndex, monitor.id, conflictingFields))
+    private fun shouldPerformPercolateQueryAndFlushInMemoryDocs(
+        numDocs: Int,
+        monitorCtx: MonitorRunnerExecutionContext,
+    ): Boolean {
+        return isInMemoryDocsSizeExceedingMemoryLimit(docsSizeOfBatchInBytes, monitorCtx) ||
+            isInMemoryNumDocsExceedingMaxDocsPerPercolateQueryLimit(numDocs, monitorCtx)
+    }
+
+    private suspend fun performPercolateQueryAndResetCounters(
+        monitorCtx: MonitorRunnerExecutionContext,
+        monitor: Monitor,
+        monitorMetadata: MonitorMetadata,
+        monitorInputIndices: List<String>,
+        concreteIndices: List<String>,
+        inputRunResults: MutableMap<String, MutableSet<String>>,
+        docsToQueries: MutableMap<String, MutableList<String>>,
+    ) {
+        try {
+            val percolateQueryResponseHits = runPercolateQueryOnTransformedDocs(
+                monitorCtx,
+                transformedDocs,
+                monitor,
+                monitorMetadata,
+                concreteIndices,
+                monitorInputIndices,
+            )
+
+            percolateQueryResponseHits.forEach { hit ->
+                var id = hit.id
+                concreteIndices.forEach { id = id.replace("_${it}_${monitor.id}", "") }
+                monitorInputIndices.forEach { id = id.replace("_${it}_${monitor.id}", "") }
+                val docIndices = hit.field("_percolator_document_slot").values.map { it.toString().toInt() }
+                docIndices.forEach { idx ->
+                    val docIndex = "${transformedDocs[idx].first}|${transformedDocs[idx].second.concreteIndexName}"
+                    inputRunResults.getOrPut(id) { mutableSetOf() }.add(docIndex)
+                    docsToQueries.getOrPut(docIndex) { mutableListOf() }.add(id)
                 }
-            } catch (e: Exception) {
-                logger.warn("Failed to run for shard $shard. Error: ${e.message}")
             }
+            totalDocsQueriedStat += transformedDocs.size.toLong()
+        } finally {
+            transformedDocs.clear()
+            docsSizeOfBatchInBytes = 0
         }
-        return matchingDocs
     }
 
+    /** Executes search query on given shard of given index to fetch docs with sequene number greater than prevSeqNo.
+     * This method hence fetches only docs from shard which haven't been queried before
+     */
     private suspend fun searchShard(
         monitorCtx: MonitorRunnerExecutionContext,
         index: String,
         shard: String,
         prevSeqNo: Long?,
         maxSeqNo: Long,
-        query: String?,
-        docIds: List<String>? = null
+        docIds: List<String>? = null,
+        fieldsToFetch: List<String>,
     ): SearchHits {
         if (prevSeqNo?.equals(maxSeqNo) == true && maxSeqNo != 0L) {
             return SearchHits.empty()
@@ -627,10 +828,6 @@ object DocumentLevelMonitorRunner : MonitorRunner() {
         val boolQueryBuilder = BoolQueryBuilder()
         boolQueryBuilder.filter(QueryBuilders.rangeQuery("_seq_no").gt(prevSeqNo).lte(maxSeqNo))
 
-        if (query != null) {
-            boolQueryBuilder.must(QueryBuilders.queryStringQuery(query))
-        }
-
         if (!docIds.isNullOrEmpty()) {
             boolQueryBuilder.filter(QueryBuilders.termsQuery("_id", docIds))
         }
@@ -641,47 +838,66 @@ object DocumentLevelMonitorRunner : MonitorRunner() {
             .source(
                 SearchSourceBuilder()
                     .version(true)
+                    .sort("_seq_no", SortOrder.DESC)
+                    .seqNoAndPrimaryTerm(true)
                     .query(boolQueryBuilder)
-                    .size(10000) // fixme: make this configurable.
+                    .size(monitorCtx.docLevelMonitorShardFetchSize)
             )
             .preference(Preference.PRIMARY_FIRST.type())
+
+        if (monitorCtx.fetchOnlyQueryFieldNames && fieldsToFetch.isNotEmpty()) {
+            request.source().fetchSource(false)
+            for (field in fieldsToFetch) {
+                request.source().fetchField(field)
+            }
+        }
         val response: SearchResponse = monitorCtx.client!!.suspendUntil { monitorCtx.client!!.search(request, it) }
         if (response.status() !== RestStatus.OK) {
-            throw IOException("Failed to search shard: $shard")
+            throw IOException("Failed to search shard: [$shard] in index [$index]. Response status is ${response.status()}")
         }
+        nonPercolateSearchesTimeTakenStat += response.took.millis
         return response.hits
     }
 
-    private suspend fun getMatchedQueries(
+    /** Executes percolate query on the docs against the monitor's query index and return the hits from the search response*/
+    private suspend fun runPercolateQueryOnTransformedDocs(
         monitorCtx: MonitorRunnerExecutionContext,
-        docs: List<BytesReference>,
+        docs: MutableList<Pair<String, TransformedDocDto>>,
         monitor: Monitor,
         monitorMetadata: MonitorMetadata,
-        index: String,
-        concreteIndex: String
+        concreteIndices: List<String>,
+        monitorInputIndices: List<String>,
     ): SearchHits {
-        val boolQueryBuilder = BoolQueryBuilder().must(QueryBuilders.matchQuery("index", index).operator(Operator.AND))
-
-        val percolateQueryBuilder = PercolateQueryBuilderExt("query", docs, XContentType.JSON)
+        val indices = docs.stream().map { it.second.indexName }.distinct().collect(Collectors.toList())
+        val boolQueryBuilder = BoolQueryBuilder().must(buildShouldClausesOverPerIndexMatchQueries(indices))
+        val percolateQueryBuilder =
+            PercolateQueryBuilderExt("query", docs.map { it.second.docSource }, XContentType.JSON)
         if (monitor.id.isNotEmpty()) {
             boolQueryBuilder.must(QueryBuilders.matchQuery("monitor_id", monitor.id).operator(Operator.AND))
         }
         boolQueryBuilder.filter(percolateQueryBuilder)
-
-        val queryIndex = monitorMetadata.sourceToQueryIndexMapping[index + monitor.id]
-        if (queryIndex == null) {
-            val message = "Failed to resolve concrete queryIndex from sourceIndex during monitor execution!" +
-                " sourceIndex:$concreteIndex queryIndex:${monitor.dataSources.queryIndex}"
+        val queryIndices =
+            docs.map { monitorMetadata.sourceToQueryIndexMapping[it.second.indexName + monitor.id] }.distinct()
+        if (queryIndices.isEmpty()) {
+            val message =
+                "Monitor ${monitor.id}: Failed to resolve query Indices from source indices during monitor execution!" +
+                    " sourceIndices: $monitorInputIndices"
             logger.error(message)
             throw AlertingException.wrap(
                 OpenSearchStatusException(message, RestStatus.INTERNAL_SERVER_ERROR)
             )
         }
-        val searchRequest = SearchRequest(queryIndex).preference(Preference.PRIMARY_FIRST.type())
+
+        val searchRequest =
+            SearchRequest().indices(*queryIndices.toTypedArray()).preference(Preference.PRIMARY_FIRST.type())
         val searchSourceBuilder = SearchSourceBuilder()
         searchSourceBuilder.query(boolQueryBuilder)
         searchRequest.source(searchSourceBuilder)
-
+        logger.debug(
+            "Monitor ${monitor.id}: " +
+                "Executing percolate query for docs from source indices " +
+                "$monitorInputIndices against query index $queryIndices"
+        )
         var response: SearchResponse
         try {
             response = monitorCtx.client!!.suspendUntil {
@@ -689,42 +905,77 @@ object DocumentLevelMonitorRunner : MonitorRunner() {
             }
         } catch (e: Exception) {
             throw IllegalStateException(
-                "Failed to run percolate search for sourceIndex [$index] and queryIndex [$queryIndex] for ${docs.size} document(s)", e
+                "Monitor ${monitor.id}:" +
+                    " Failed to run percolate search for sourceIndex [${concreteIndices.joinToString()}] " +
+                    "and queryIndex [${queryIndices.joinToString()}] for ${docs.size} document(s)",
+                e
             )
         }
 
         if (response.status() !== RestStatus.OK) {
-            throw IOException("Failed to search percolate index: $queryIndex")
+            throw IOException(
+                "Monitor ${monitor.id}: Failed to search percolate index: ${queryIndices.joinToString()}. " +
+                    "Response status is ${response.status()}"
+            )
         }
+        logger.debug("Monitor ${monitor.id} PERF_DEBUG: Percolate query time taken millis = ${response.took}")
+        percolateQueriesTimeTakenStat += response.took.millis
         return response.hits
     }
+    /** we cannot use terms query because `index` field's mapping is of type TEXT and not keyword. Refer doc-level-queries.json*/
+    private fun buildShouldClausesOverPerIndexMatchQueries(indices: List<String>): BoolQueryBuilder {
+        val boolQueryBuilder = QueryBuilders.boolQuery()
+        indices.forEach { boolQueryBuilder.should(QueryBuilders.matchQuery("index", it)) }
+        return boolQueryBuilder
+    }
 
-    private fun getAllDocs(
+    /** Transform field names and index names in all the search hits to format required to run percolate search against them.
+     * Hits are transformed using method transformDocumentFieldNames() */
+    private fun transformSearchHitsAndReconstructDocs(
         hits: SearchHits,
         index: String,
         concreteIndex: String,
         monitorId: String,
-        conflictingFields: List<String>
-    ): List<Pair<String, BytesReference>> {
-        return hits.map { hit ->
-            val sourceMap = hit.sourceAsMap
-
-            transformDocumentFieldNames(
-                sourceMap,
-                conflictingFields,
-                "_${index}_$monitorId",
-                "_${concreteIndex}_$monitorId",
-                ""
-            )
-
-            var xContentBuilder = XContentFactory.jsonBuilder().map(sourceMap)
-
-            val sourceRef = BytesReference.bytes(xContentBuilder)
-
-            logger.debug("Document [${hit.id}] payload after transform: ", sourceRef.utf8ToString())
+        conflictingFields: List<String>,
+    ): List<Pair<String, TransformedDocDto>> {
+        return hits.mapNotNull(fun(hit: SearchHit): Pair<String, TransformedDocDto>? {
+            try {
+                val sourceMap = if (hit.hasSource()) {
+                    hit.sourceAsMap
+                } else {
+                    constructSourceMapFromFieldsInHit(hit)
+                }
+                transformDocumentFieldNames(
+                    sourceMap,
+                    conflictingFields,
+                    "_${index}_$monitorId",
+                    "_${concreteIndex}_$monitorId",
+                    ""
+                )
+                var xContentBuilder = XContentFactory.jsonBuilder().map(sourceMap)
+                val sourceRef = BytesReference.bytes(xContentBuilder)
+                docsSizeOfBatchInBytes += sourceRef.ramBytesUsed()
+                totalDocsSizeInBytesStat += sourceRef.ramBytesUsed()
+                return Pair(hit.id, TransformedDocDto(index, concreteIndex, hit.id, sourceRef))
+            } catch (e: Exception) {
+                logger.error("Monitor $monitorId: Failed to transform payload $hit for percolate query", e)
+                // skip any document which we fail to transform because we anyway won't be able to run percolate queries on them.
+                return null
+            }
+        })
+    }
 
-            Pair(hit.id, sourceRef)
+    private fun constructSourceMapFromFieldsInHit(hit: SearchHit): MutableMap<String, Any> {
+        if (hit.fields == null)
+            return mutableMapOf()
+        val sourceMap: MutableMap<String, Any> = mutableMapOf()
+        for (field in hit.fields) {
+            if (field.value.values != null && field.value.values.isNotEmpty())
+                if (field.value.values.size == 1) {
+                    sourceMap[field.key] = field.value.values[0]
+                } else sourceMap[field.key] = field.value.values
         }
+        return sourceMap
     }
 
     /**
@@ -777,4 +1028,33 @@ object DocumentLevelMonitorRunner : MonitorRunner() {
         }
         jsonAsMap.putAll(tempMap)
     }
+
+    /**
+     * Returns true, if the docs fetched from shards thus far amount to less than threshold
+     * amount of percentage (default:10. setting is dynamic and configurable) of the total heap size or not.
+     *
+     */
+    private fun isInMemoryDocsSizeExceedingMemoryLimit(docsBytesSize: Long, monitorCtx: MonitorRunnerExecutionContext): Boolean {
+        var thresholdPercentage = monitorCtx.percQueryDocsSizeMemoryPercentageLimit
+        val heapMaxBytes = monitorCtx.jvmStats!!.mem.heapMax.bytes
+        val thresholdBytes = (thresholdPercentage.toDouble() / 100.0) * heapMaxBytes
+
+        return docsBytesSize > thresholdBytes
+    }
+
+    private fun isInMemoryNumDocsExceedingMaxDocsPerPercolateQueryLimit(numDocs: Int, monitorCtx: MonitorRunnerExecutionContext): Boolean {
+        var maxNumDocsThreshold = monitorCtx.percQueryMaxNumDocsInMemory
+        return numDocs >= maxNumDocsThreshold
+    }
+
+    /**
+     * POJO holding information about each doc's concrete index, id, input index pattern/alias/datastream name
+     * and doc source. A list of these POJOs would be passed to percolate query execution logic.
+     */
+    data class TransformedDocDto(
+        var indexName: String,
+        var concreteIndexName: String,
+        var docId: String,
+        var docSource: BytesReference
+    )
 }
diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/InputService.kt b/alerting/src/main/kotlin/org/opensearch/alerting/InputService.kt
index b31e21d5f..17ec4670c 100644
--- a/alerting/src/main/kotlin/org/opensearch/alerting/InputService.kt
+++ b/alerting/src/main/kotlin/org/opensearch/alerting/InputService.kt
@@ -5,6 +5,9 @@
 
 package org.opensearch.alerting
 
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.launch
 import org.apache.logging.log4j.LogManager
 import org.opensearch.action.search.SearchRequest
 import org.opensearch.action.search.SearchResponse
@@ -12,7 +15,9 @@ import org.opensearch.alerting.model.InputRunResults
 import org.opensearch.alerting.model.TriggerAfterKey
 import org.opensearch.alerting.opensearchapi.convertToMap
 import org.opensearch.alerting.opensearchapi.suspendUntil
+import org.opensearch.alerting.settings.AlertingSettings
 import org.opensearch.alerting.util.AggregationQueryRewriter
+import org.opensearch.alerting.util.CrossClusterMonitorUtils
 import org.opensearch.alerting.util.addUserBackendRolesFilter
 import org.opensearch.alerting.util.clusterMetricsMonitorHelpers.executeTransportAction
 import org.opensearch.alerting.util.clusterMetricsMonitorHelpers.toMap
@@ -43,6 +48,8 @@ import org.opensearch.script.TemplateScript
 import org.opensearch.search.builder.SearchSourceBuilder
 import java.time.Instant
 
+private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO)
+
 /** Service that handles the collection of input results for Monitor executions */
 class InputService(
     val client: Client,
@@ -100,8 +107,9 @@ class InputService(
                             .newInstance(searchParams)
                             .execute()
 
+                        val indexes = CrossClusterMonitorUtils.parseIndexesForRemoteSearch(input.indices, clusterService)
                         val searchRequest = SearchRequest()
-                            .indices(*input.indices.toTypedArray())
+                            .indices(*indexes.toTypedArray())
                             .preference(Preference.PRIMARY_FIRST.type())
                         XContentType.JSON.xContent().createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, searchSource).use {
                             searchRequest.source(SearchSourceBuilder.fromXContent(it))
@@ -115,9 +123,36 @@ class InputService(
                         results += searchResponse.convertToMap()
                     }
                     is ClusterMetricsInput -> {
-                        logger.debug("ClusterMetricsInput clusterMetricType: ${input.clusterMetricType}")
-                        val response = executeTransportAction(input, client)
-                        results += response.toMap()
+                        logger.debug("ClusterMetricsInput clusterMetricType: {}", input.clusterMetricType)
+
+                        val remoteMonitoringEnabled = clusterService.clusterSettings.get(AlertingSettings.REMOTE_MONITORING_ENABLED)
+                        logger.debug("Remote monitoring enabled: {}", remoteMonitoringEnabled)
+
+                        val responseMap = mutableMapOf<String, Map<String, Any>>()
+                        if (remoteMonitoringEnabled && input.clusters.isNotEmpty()) {
+                            client.threadPool().threadContext.stashContext().use {
+                                scope.launch {
+                                    input.clusters.forEach { cluster ->
+                                        val targetClient = CrossClusterMonitorUtils.getClientForCluster(cluster, client, clusterService)
+                                        val response = executeTransportAction(input, targetClient)
+                                        // Not all supported API reference the cluster name in their response.
+                                        // Mapping each response to the cluster name before adding to results.
+                                        // Not adding this same logic for local-only monitors to avoid breaking existing monitors.
+                                        responseMap[cluster] = response.toMap()
+                                    }
+                                }
+                            }
+                            val inputTimeout = clusterService.clusterSettings.get(AlertingSettings.INPUT_TIMEOUT)
+                            val startTime = Instant.now().toEpochMilli()
+                            while (
+                                (Instant.now().toEpochMilli() - startTime >= inputTimeout.millis) ||
+                                (responseMap.size < input.clusters.size)
+                            ) { /* Wait for responses */ }
+                            results += responseMap
+                        } else {
+                            val response = executeTransportAction(input, client)
+                            results += response.toMap()
+                        }
                     }
                     else -> {
                         throw IllegalArgumentException("Unsupported input type: ${input.name()}.")
diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/MonitorMetadataService.kt b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorMetadataService.kt
index e4c5a318c..da167ed49 100644
--- a/alerting/src/main/kotlin/org/opensearch/alerting/MonitorMetadataService.kt
+++ b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorMetadataService.kt
@@ -12,6 +12,7 @@ import kotlinx.coroutines.SupervisorJob
 import org.apache.logging.log4j.LogManager
 import org.opensearch.ExceptionsHelper
 import org.opensearch.OpenSearchSecurityException
+import org.opensearch.OpenSearchStatusException
 import org.opensearch.action.DocWriteRequest
 import org.opensearch.action.DocWriteResponse
 import org.opensearch.action.admin.indices.get.GetIndexRequest
@@ -28,6 +29,7 @@ import org.opensearch.alerting.model.MonitorMetadata
 import org.opensearch.alerting.opensearchapi.suspendUntil
 import org.opensearch.alerting.settings.AlertingSettings
 import org.opensearch.alerting.util.AlertingException
+import org.opensearch.alerting.util.IndexUtils
 import org.opensearch.client.Client
 import org.opensearch.cluster.service.ClusterService
 import org.opensearch.common.settings.Settings
@@ -77,35 +79,51 @@ object MonitorMetadataService :
     @Suppress("ComplexMethod", "ReturnCount")
     suspend fun upsertMetadata(metadata: MonitorMetadata, updating: Boolean): MonitorMetadata {
         try {
-            val indexRequest = IndexRequest(ScheduledJob.SCHEDULED_JOBS_INDEX)
-                .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
-                .source(metadata.toXContent(XContentFactory.jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true"))))
-                .id(metadata.id)
-                .routing(metadata.monitorId)
-                .setIfSeqNo(metadata.seqNo)
-                .setIfPrimaryTerm(metadata.primaryTerm)
-                .timeout(indexTimeout)
+            if (clusterService.state().routingTable.hasIndex(ScheduledJob.SCHEDULED_JOBS_INDEX)) {
+                val indexRequest = IndexRequest(ScheduledJob.SCHEDULED_JOBS_INDEX)
+                    .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+                    .source(
+                        metadata.toXContent(
+                            XContentFactory.jsonBuilder(),
+                            ToXContent.MapParams(mapOf("with_type" to "true"))
+                        )
+                    )
+                    .id(metadata.id)
+                    .routing(metadata.monitorId)
+                    .setIfSeqNo(metadata.seqNo)
+                    .setIfPrimaryTerm(metadata.primaryTerm)
+                    .timeout(indexTimeout)
 
-            if (updating) {
-                indexRequest.id(metadata.id).setIfSeqNo(metadata.seqNo).setIfPrimaryTerm(metadata.primaryTerm)
-            } else {
-                indexRequest.opType(DocWriteRequest.OpType.CREATE)
-            }
-            val response: IndexResponse = client.suspendUntil { index(indexRequest, it) }
-            when (response.result) {
-                DocWriteResponse.Result.DELETED, DocWriteResponse.Result.NOOP, DocWriteResponse.Result.NOT_FOUND, null -> {
-                    val failureReason = "The upsert metadata call failed with a ${response.result?.lowercase} result"
-                    log.error(failureReason)
-                    throw AlertingException(failureReason, RestStatus.INTERNAL_SERVER_ERROR, IllegalStateException(failureReason))
+                if (updating) {
+                    indexRequest.id(metadata.id).setIfSeqNo(metadata.seqNo).setIfPrimaryTerm(metadata.primaryTerm)
+                } else {
+                    indexRequest.opType(DocWriteRequest.OpType.CREATE)
                 }
-                DocWriteResponse.Result.CREATED, DocWriteResponse.Result.UPDATED -> {
-                    log.debug("Successfully upserted MonitorMetadata:${metadata.id} ")
+                val response: IndexResponse = client.suspendUntil { index(indexRequest, it) }
+                when (response.result) {
+                    DocWriteResponse.Result.DELETED, DocWriteResponse.Result.NOOP, DocWriteResponse.Result.NOT_FOUND, null -> {
+                        val failureReason =
+                            "The upsert metadata call failed with a ${response.result?.lowercase} result"
+                        log.error(failureReason)
+                        throw AlertingException(
+                            failureReason,
+                            RestStatus.INTERNAL_SERVER_ERROR,
+                            IllegalStateException(failureReason)
+                        )
+                    }
+
+                    DocWriteResponse.Result.CREATED, DocWriteResponse.Result.UPDATED -> {
+                        log.debug("Successfully upserted MonitorMetadata:${metadata.id} ")
+                    }
                 }
+                return metadata.copy(
+                    seqNo = response.seqNo,
+                    primaryTerm = response.primaryTerm
+                )
+            } else {
+                val failureReason = "Job index ${ScheduledJob.SCHEDULED_JOBS_INDEX} does not exist to update monitor metadata"
+                throw OpenSearchStatusException(failureReason, RestStatus.INTERNAL_SERVER_ERROR)
             }
-            return metadata.copy(
-                seqNo = response.seqNo,
-                primaryTerm = response.primaryTerm
-            )
         } catch (e: Exception) {
             throw AlertingException.wrap(e)
         }
@@ -216,11 +234,19 @@ object MonitorMetadataService :
         val lastRunContext = existingRunContext?.toMutableMap() ?: mutableMapOf()
         try {
             if (index == null) return mutableMapOf()
-            val getIndexRequest = GetIndexRequest().indices(index)
-            val getIndexResponse: GetIndexResponse = client.suspendUntil {
-                client.admin().indices().getIndex(getIndexRequest, it)
+
+            val indices = mutableListOf<String>()
+            if (IndexUtils.isAlias(index, clusterService.state()) ||
+                IndexUtils.isDataStream(index, clusterService.state())
+            ) {
+                IndexUtils.getWriteIndex(index, clusterService.state())?.let { indices.add(it) }
+            } else {
+                val getIndexRequest = GetIndexRequest().indices(index)
+                val getIndexResponse: GetIndexResponse = client.suspendUntil {
+                    client.admin().indices().getIndex(getIndexRequest, it)
+                }
+                indices.addAll(getIndexResponse.indices())
             }
-            val indices = getIndexResponse.indices()
 
             indices.forEach { indexName ->
                 if (!lastRunContext.containsKey(indexName)) {
diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerExecutionContext.kt b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerExecutionContext.kt
index 41a26bb79..2aed10a9a 100644
--- a/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerExecutionContext.kt
+++ b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerExecutionContext.kt
@@ -7,6 +7,7 @@ package org.opensearch.alerting
 
 import org.opensearch.action.bulk.BackoffPolicy
 import org.opensearch.alerting.alerts.AlertIndices
+import org.opensearch.alerting.core.lock.LockService
 import org.opensearch.alerting.model.destination.DestinationContextFactory
 import org.opensearch.alerting.settings.AlertingSettings
 import org.opensearch.alerting.settings.DestinationSettings
@@ -18,6 +19,7 @@ import org.opensearch.cluster.service.ClusterService
 import org.opensearch.common.settings.Settings
 import org.opensearch.common.unit.TimeValue
 import org.opensearch.core.xcontent.NamedXContentRegistry
+import org.opensearch.monitor.jvm.JvmStats
 import org.opensearch.script.ScriptService
 import org.opensearch.threadpool.ThreadPool
 
@@ -36,6 +38,7 @@ data class MonitorRunnerExecutionContext(
     var alertService: AlertService? = null,
     var docLevelMonitorQueries: DocLevelMonitorQueries? = null,
     var workflowService: WorkflowService? = null,
+    var jvmStats: JvmStats? = null,
 
     @Volatile var retryPolicy: BackoffPolicy? = null,
     @Volatile var moveAlertsRetryPolicy: BackoffPolicy? = null,
@@ -47,5 +50,13 @@ data class MonitorRunnerExecutionContext(
     @Volatile var destinationContextFactory: DestinationContextFactory? = null,
 
     @Volatile var maxActionableAlertCount: Long = AlertingSettings.DEFAULT_MAX_ACTIONABLE_ALERT_COUNT,
-    @Volatile var indexTimeout: TimeValue? = null
+    @Volatile var indexTimeout: TimeValue? = null,
+    @Volatile var findingsIndexBatchSize: Int = AlertingSettings.DEFAULT_FINDINGS_INDEXING_BATCH_SIZE,
+    @Volatile var fetchOnlyQueryFieldNames: Boolean = true,
+    @Volatile var percQueryMaxNumDocsInMemory: Int = AlertingSettings.DEFAULT_PERCOLATE_QUERY_NUM_DOCS_IN_MEMORY,
+    @Volatile var percQueryDocsSizeMemoryPercentageLimit: Int =
+        AlertingSettings.DEFAULT_PERCOLATE_QUERY_DOCS_SIZE_MEMORY_PERCENTAGE_LIMIT,
+    @Volatile var docLevelMonitorShardFetchSize: Int =
+        AlertingSettings.DEFAULT_DOC_LEVEL_MONITOR_SHARD_FETCH_SIZE,
+    @Volatile var lockService: LockService? = null
 )
diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerService.kt b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerService.kt
index ca223f7a0..0763bcae4 100644
--- a/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerService.kt
+++ b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerService.kt
@@ -17,17 +17,26 @@ import org.opensearch.alerting.alerts.AlertIndices
 import org.opensearch.alerting.alerts.AlertMover.Companion.moveAlerts
 import org.opensearch.alerting.core.JobRunner
 import org.opensearch.alerting.core.ScheduledJobIndices
+import org.opensearch.alerting.core.lock.LockModel
+import org.opensearch.alerting.core.lock.LockService
 import org.opensearch.alerting.model.MonitorRunResult
 import org.opensearch.alerting.model.WorkflowRunResult
 import org.opensearch.alerting.model.destination.DestinationContextFactory
 import org.opensearch.alerting.opensearchapi.retry
+import org.opensearch.alerting.opensearchapi.suspendUntil
 import org.opensearch.alerting.script.TriggerExecutionContext
+import org.opensearch.alerting.settings.AlertingSettings
 import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERT_BACKOFF_COUNT
 import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERT_BACKOFF_MILLIS
+import org.opensearch.alerting.settings.AlertingSettings.Companion.DOC_LEVEL_MONITOR_FETCH_ONLY_QUERY_FIELDS_ENABLED
+import org.opensearch.alerting.settings.AlertingSettings.Companion.DOC_LEVEL_MONITOR_SHARD_FETCH_SIZE
+import org.opensearch.alerting.settings.AlertingSettings.Companion.FINDINGS_INDEXING_BATCH_SIZE
 import org.opensearch.alerting.settings.AlertingSettings.Companion.INDEX_TIMEOUT
 import org.opensearch.alerting.settings.AlertingSettings.Companion.MAX_ACTIONABLE_ALERT_COUNT
 import org.opensearch.alerting.settings.AlertingSettings.Companion.MOVE_ALERTS_BACKOFF_COUNT
 import org.opensearch.alerting.settings.AlertingSettings.Companion.MOVE_ALERTS_BACKOFF_MILLIS
+import org.opensearch.alerting.settings.AlertingSettings.Companion.PERCOLATE_QUERY_DOCS_SIZE_MEMORY_PERCENTAGE_LIMIT
+import org.opensearch.alerting.settings.AlertingSettings.Companion.PERCOLATE_QUERY_MAX_NUM_DOCS_IN_MEMORY
 import org.opensearch.alerting.settings.DestinationSettings.Companion.ALLOW_LIST
 import org.opensearch.alerting.settings.DestinationSettings.Companion.HOST_DENY_LIST
 import org.opensearch.alerting.settings.DestinationSettings.Companion.loadDestinationSettings
@@ -48,6 +57,7 @@ import org.opensearch.commons.alerting.model.action.Action
 import org.opensearch.commons.alerting.util.isBucketLevelMonitor
 import org.opensearch.core.action.ActionListener
 import org.opensearch.core.xcontent.NamedXContentRegistry
+import org.opensearch.monitor.jvm.JvmStats
 import org.opensearch.script.Script
 import org.opensearch.script.ScriptService
 import org.opensearch.script.TemplateScript
@@ -132,6 +142,11 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon
         return this
     }
 
+    fun registerJvmStats(jvmStats: JvmStats): MonitorRunnerService {
+        this.monitorCtx.jvmStats = jvmStats
+        return this
+    }
+
     // Must be called after registerClusterService and registerSettings in AlertingPlugin
     fun registerConsumers(): MonitorRunnerService {
         monitorCtx.retryPolicy = BackoffPolicy.constantBackoff(
@@ -169,6 +184,35 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon
 
         monitorCtx.indexTimeout = INDEX_TIMEOUT.get(monitorCtx.settings)
 
+        monitorCtx.findingsIndexBatchSize = FINDINGS_INDEXING_BATCH_SIZE.get(monitorCtx.settings)
+        monitorCtx.clusterService!!.clusterSettings.addSettingsUpdateConsumer(AlertingSettings.FINDINGS_INDEXING_BATCH_SIZE) {
+            monitorCtx.findingsIndexBatchSize = it
+        }
+
+        monitorCtx.fetchOnlyQueryFieldNames = DOC_LEVEL_MONITOR_FETCH_ONLY_QUERY_FIELDS_ENABLED.get(monitorCtx.settings)
+        monitorCtx.clusterService!!.clusterSettings.addSettingsUpdateConsumer(DOC_LEVEL_MONITOR_FETCH_ONLY_QUERY_FIELDS_ENABLED) {
+            monitorCtx.fetchOnlyQueryFieldNames = it
+        }
+
+        monitorCtx.percQueryMaxNumDocsInMemory = PERCOLATE_QUERY_MAX_NUM_DOCS_IN_MEMORY.get(monitorCtx.settings)
+        monitorCtx.clusterService!!.clusterSettings.addSettingsUpdateConsumer(PERCOLATE_QUERY_MAX_NUM_DOCS_IN_MEMORY) {
+            monitorCtx.percQueryMaxNumDocsInMemory = it
+        }
+
+        monitorCtx.percQueryDocsSizeMemoryPercentageLimit =
+            PERCOLATE_QUERY_DOCS_SIZE_MEMORY_PERCENTAGE_LIMIT.get(monitorCtx.settings)
+        monitorCtx.clusterService!!.clusterSettings
+            .addSettingsUpdateConsumer(PERCOLATE_QUERY_DOCS_SIZE_MEMORY_PERCENTAGE_LIMIT) {
+                monitorCtx.percQueryDocsSizeMemoryPercentageLimit = it
+            }
+
+        monitorCtx.docLevelMonitorShardFetchSize =
+            DOC_LEVEL_MONITOR_SHARD_FETCH_SIZE.get(monitorCtx.settings)
+        monitorCtx.clusterService!!.clusterSettings
+            .addSettingsUpdateConsumer(DOC_LEVEL_MONITOR_SHARD_FETCH_SIZE) {
+                monitorCtx.docLevelMonitorShardFetchSize = it
+            }
+
         return this
     }
 
@@ -180,6 +224,11 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon
         return this
     }
 
+    fun registerLockService(lockService: LockService): MonitorRunnerService {
+        monitorCtx.lockService = lockService
+        return this
+    }
+
     // Updates destination settings when the reload API is called so that new keystore values are visible
     fun reloadDestinationSettings(settings: Settings) {
         monitorCtx.destinationSettings = loadDestinationSettings(settings)
@@ -251,12 +300,40 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon
         when (job) {
             is Workflow -> {
                 launch {
-                    runJob(job, periodStart, periodEnd, false)
+                    var lock: LockModel? = null
+                    try {
+                        lock = monitorCtx.client!!.suspendUntil<Client, LockModel?> {
+                            monitorCtx.lockService!!.acquireLock(job, it)
+                        } ?: return@launch
+                        logger.debug("lock ${lock!!.lockId} acquired")
+                        logger.debug(
+                            "PERF_DEBUG: executing workflow ${job.id} on node " +
+                                monitorCtx.clusterService!!.state().nodes().localNode.id
+                        )
+                        runJob(job, periodStart, periodEnd, false)
+                    } finally {
+                        monitorCtx.client!!.suspendUntil<Client, Boolean> { monitorCtx.lockService!!.release(lock, it) }
+                        logger.debug("lock ${lock!!.lockId} released")
+                    }
                 }
             }
             is Monitor -> {
                 launch {
-                    runJob(job, periodStart, periodEnd, false)
+                    var lock: LockModel? = null
+                    try {
+                        lock = monitorCtx.client!!.suspendUntil<Client, LockModel?> {
+                            monitorCtx.lockService!!.acquireLock(job, it)
+                        } ?: return@launch
+                        logger.debug("lock ${lock!!.lockId} acquired")
+                        logger.debug(
+                            "PERF_DEBUG: executing ${job.monitorType} ${job.id} on node " +
+                                monitorCtx.clusterService!!.state().nodes().localNode.id
+                        )
+                        runJob(job, periodStart, periodEnd, false)
+                    } finally {
+                        monitorCtx.client!!.suspendUntil<Client, Boolean> { monitorCtx.lockService!!.release(lock, it) }
+                        logger.debug("lock ${lock!!.lockId} released")
+                    }
                 }
             }
             else -> {
@@ -300,7 +377,7 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon
         val runResult = if (monitor.isBucketLevelMonitor()) {
             BucketLevelMonitorRunner.runMonitor(monitor, monitorCtx, periodStart, periodEnd, dryrun, executionId = executionId)
         } else if (monitor.isDocLevelMonitor()) {
-            DocumentLevelMonitorRunner.runMonitor(monitor, monitorCtx, periodStart, periodEnd, dryrun, executionId = executionId)
+            DocumentLevelMonitorRunner().runMonitor(monitor, monitorCtx, periodStart, periodEnd, dryrun, executionId = executionId)
         } else {
             QueryLevelMonitorRunner.runMonitor(monitor, monitorCtx, periodStart, periodEnd, dryrun, executionId = executionId)
         }
diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/QueryLevelMonitorRunner.kt b/alerting/src/main/kotlin/org/opensearch/alerting/QueryLevelMonitorRunner.kt
index 691071517..a77121069 100644
--- a/alerting/src/main/kotlin/org/opensearch/alerting/QueryLevelMonitorRunner.kt
+++ b/alerting/src/main/kotlin/org/opensearch/alerting/QueryLevelMonitorRunner.kt
@@ -11,6 +11,7 @@ import org.opensearch.alerting.model.QueryLevelTriggerRunResult
 import org.opensearch.alerting.opensearchapi.InjectorContextElement
 import org.opensearch.alerting.opensearchapi.withClosableContext
 import org.opensearch.alerting.script.QueryLevelTriggerExecutionContext
+import org.opensearch.alerting.settings.AlertingSettings
 import org.opensearch.alerting.util.isADMonitor
 import org.opensearch.alerting.workflow.WorkflowRunContext
 import org.opensearch.commons.alerting.model.Alert
@@ -65,7 +66,21 @@ object QueryLevelMonitorRunner : MonitorRunner() {
         for (trigger in monitor.triggers) {
             val currentAlert = currentAlerts[trigger]
             val triggerCtx = QueryLevelTriggerExecutionContext(monitor, trigger as QueryLevelTrigger, monitorResult, currentAlert)
-            val triggerResult = monitorCtx.triggerService!!.runQueryLevelTrigger(monitor, trigger, triggerCtx)
+            val triggerResult = when (monitor.monitorType) {
+                Monitor.MonitorType.QUERY_LEVEL_MONITOR ->
+                    monitorCtx.triggerService!!.runQueryLevelTrigger(monitor, trigger, triggerCtx)
+                Monitor.MonitorType.CLUSTER_METRICS_MONITOR -> {
+                    val remoteMonitoringEnabled =
+                        monitorCtx.clusterService!!.clusterSettings.get(AlertingSettings.REMOTE_MONITORING_ENABLED)
+                    logger.debug("Remote monitoring enabled: {}", remoteMonitoringEnabled)
+                    if (remoteMonitoringEnabled)
+                        monitorCtx.triggerService!!.runClusterMetricsTrigger(monitor, trigger, triggerCtx, monitorCtx.clusterService!!)
+                    else monitorCtx.triggerService!!.runQueryLevelTrigger(monitor, trigger, triggerCtx)
+                }
+                else ->
+                    throw IllegalArgumentException("Unsupported monitor type: ${monitor.monitorType.name}.")
+            }
+
             triggerResults[trigger.id] = triggerResult
 
             if (monitorCtx.triggerService!!.isQueryLevelTriggerActionable(triggerCtx, triggerResult, workflowRunContext)) {
diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/TriggerService.kt b/alerting/src/main/kotlin/org/opensearch/alerting/TriggerService.kt
index f2356eddf..21ba32475 100644
--- a/alerting/src/main/kotlin/org/opensearch/alerting/TriggerService.kt
+++ b/alerting/src/main/kotlin/org/opensearch/alerting/TriggerService.kt
@@ -9,6 +9,8 @@ import org.apache.logging.log4j.LogManager
 import org.opensearch.alerting.chainedAlertCondition.parsers.ChainedAlertExpressionParser
 import org.opensearch.alerting.model.BucketLevelTriggerRunResult
 import org.opensearch.alerting.model.ChainedAlertTriggerRunResult
+import org.opensearch.alerting.model.ClusterMetricsTriggerRunResult
+import org.opensearch.alerting.model.ClusterMetricsTriggerRunResult.ClusterTriggerResult
 import org.opensearch.alerting.model.DocumentLevelTriggerRunResult
 import org.opensearch.alerting.model.QueryLevelTriggerRunResult
 import org.opensearch.alerting.script.BucketLevelTriggerExecutionContext
@@ -16,8 +18,10 @@ import org.opensearch.alerting.script.ChainedAlertTriggerExecutionContext
 import org.opensearch.alerting.script.QueryLevelTriggerExecutionContext
 import org.opensearch.alerting.script.TriggerScript
 import org.opensearch.alerting.triggercondition.parsers.TriggerExpressionParser
+import org.opensearch.alerting.util.CrossClusterMonitorUtils
 import org.opensearch.alerting.util.getBucketKeysHash
 import org.opensearch.alerting.workflow.WorkflowRunContext
+import org.opensearch.cluster.service.ClusterService
 import org.opensearch.commons.alerting.aggregation.bucketselectorext.BucketSelectorIndices.Fields.BUCKET_INDICES
 import org.opensearch.commons.alerting.aggregation.bucketselectorext.BucketSelectorIndices.Fields.PARENT_BUCKET_PATH
 import org.opensearch.commons.alerting.model.AggregationResultBucket
@@ -79,6 +83,52 @@ class TriggerService(val scriptService: ScriptService) {
         }
     }
 
+    fun runClusterMetricsTrigger(
+        monitor: Monitor,
+        trigger: QueryLevelTrigger,
+        ctx: QueryLevelTriggerExecutionContext,
+        clusterService: ClusterService
+    ): ClusterMetricsTriggerRunResult {
+        var runResult: ClusterMetricsTriggerRunResult?
+        try {
+            val inputResults = ctx.results.getOrElse(0) { mapOf() }
+            var triggered = false
+            val clusterTriggerResults = mutableListOf<ClusterTriggerResult>()
+            if (CrossClusterMonitorUtils.isRemoteMonitor(monitor, clusterService)) {
+                inputResults.forEach { clusterResult ->
+                    // Reducing the inputResults to only include results from 1 cluster at a time
+                    val clusterTriggerCtx = ctx.copy(results = listOf(mapOf(clusterResult.toPair())))
+
+                    val clusterTriggered = scriptService.compile(trigger.condition, TriggerScript.CONTEXT)
+                        .newInstance(trigger.condition.params)
+                        .execute(clusterTriggerCtx)
+
+                    if (clusterTriggered) {
+                        triggered = clusterTriggered
+                        clusterTriggerResults.add(ClusterTriggerResult(cluster = clusterResult.key, triggered = clusterTriggered))
+                    }
+                }
+            } else {
+                triggered = scriptService.compile(trigger.condition, TriggerScript.CONTEXT)
+                    .newInstance(trigger.condition.params)
+                    .execute(ctx)
+                if (triggered) clusterTriggerResults
+                    .add(ClusterTriggerResult(cluster = clusterService.clusterName.value(), triggered = triggered))
+            }
+            runResult = ClusterMetricsTriggerRunResult(
+                triggerName = trigger.name,
+                triggered = triggered,
+                error = null,
+                clusterTriggerResults = clusterTriggerResults
+            )
+        } catch (e: Exception) {
+            logger.info("Error running script for monitor ${monitor.id}, trigger: ${trigger.id}", e)
+            // if the script fails we need to send an alert so set triggered = true
+            runResult = ClusterMetricsTriggerRunResult(trigger.name, true, e)
+        }
+        return runResult!!
+    }
+
     // TODO: improve performance and support match all and match any
     fun runDocLevelTrigger(
         monitor: Monitor,
diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/action/GetMonitorAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/action/GetMonitorAction.kt
deleted file mode 100644
index da209b983..000000000
--- a/alerting/src/main/kotlin/org/opensearch/alerting/action/GetMonitorAction.kt
+++ /dev/null
@@ -1,15 +0,0 @@
-/*
- * Copyright OpenSearch Contributors
- * SPDX-License-Identifier: Apache-2.0
- */
-
-package org.opensearch.alerting.action
-
-import org.opensearch.action.ActionType
-
-class GetMonitorAction private constructor() : ActionType<GetMonitorResponse>(NAME, ::GetMonitorResponse) {
-    companion object {
-        val INSTANCE = GetMonitorAction()
-        const val NAME = "cluster:admin/opendistro/alerting/monitor/get"
-    }
-}
diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/action/GetMonitorRequest.kt b/alerting/src/main/kotlin/org/opensearch/alerting/action/GetMonitorRequest.kt
deleted file mode 100644
index 365a31f8f..000000000
--- a/alerting/src/main/kotlin/org/opensearch/alerting/action/GetMonitorRequest.kt
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Copyright OpenSearch Contributors
- * SPDX-License-Identifier: Apache-2.0
- */
-
-package org.opensearch.alerting.action
-
-import org.opensearch.action.ActionRequest
-import org.opensearch.action.ActionRequestValidationException
-import org.opensearch.core.common.io.stream.StreamInput
-import org.opensearch.core.common.io.stream.StreamOutput
-import org.opensearch.rest.RestRequest
-import org.opensearch.search.fetch.subphase.FetchSourceContext
-import java.io.IOException
-
-class GetMonitorRequest : ActionRequest {
-    val monitorId: String
-    val version: Long
-    val method: RestRequest.Method
-    val srcContext: FetchSourceContext?
-
-    constructor(
-        monitorId: String,
-        version: Long,
-        method: RestRequest.Method,
-        srcContext: FetchSourceContext?
-    ) : super() {
-        this.monitorId = monitorId
-        this.version = version
-        this.method = method
-        this.srcContext = srcContext
-    }
-
-    @Throws(IOException::class)
-    constructor(sin: StreamInput) : this(
-        sin.readString(), // monitorId
-        sin.readLong(), // version
-        sin.readEnum(RestRequest.Method::class.java), // method
-        if (sin.readBoolean()) {
-            FetchSourceContext(sin) // srcContext
-        } else null
-    )
-
-    override fun validate(): ActionRequestValidationException? {
-        return null
-    }
-
-    @Throws(IOException::class)
-    override fun writeTo(out: StreamOutput) {
-        out.writeString(monitorId)
-        out.writeLong(version)
-        out.writeEnum(method)
-        out.writeBoolean(srcContext != null)
-        srcContext?.writeTo(out)
-    }
-}
diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/action/GetMonitorResponse.kt b/alerting/src/main/kotlin/org/opensearch/alerting/action/GetMonitorResponse.kt
deleted file mode 100644
index ab69a1582..000000000
--- a/alerting/src/main/kotlin/org/opensearch/alerting/action/GetMonitorResponse.kt
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Copyright OpenSearch Contributors
- * SPDX-License-Identifier: Apache-2.0
- */
-
-package org.opensearch.alerting.action
-
-import org.opensearch.commons.alerting.model.Monitor
-import org.opensearch.commons.alerting.util.IndexUtils.Companion._ID
-import org.opensearch.commons.alerting.util.IndexUtils.Companion._PRIMARY_TERM
-import org.opensearch.commons.alerting.util.IndexUtils.Companion._SEQ_NO
-import org.opensearch.commons.alerting.util.IndexUtils.Companion._VERSION
-import org.opensearch.core.action.ActionResponse
-import org.opensearch.core.common.io.stream.StreamInput
-import org.opensearch.core.common.io.stream.StreamOutput
-import org.opensearch.core.rest.RestStatus
-import org.opensearch.core.xcontent.ToXContent
-import org.opensearch.core.xcontent.ToXContentFragment
-import org.opensearch.core.xcontent.ToXContentObject
-import org.opensearch.core.xcontent.XContentBuilder
-import java.io.IOException
-
-class GetMonitorResponse : ActionResponse, ToXContentObject {
-    var id: String
-    var version: Long
-    var seqNo: Long
-    var primaryTerm: Long
-    var status: RestStatus
-    var monitor: Monitor?
-    var associatedWorkflows: List<AssociatedWorkflow>?
-
-    constructor(
-        id: String,
-        version: Long,
-        seqNo: Long,
-        primaryTerm: Long,
-        status: RestStatus,
-        monitor: Monitor?,
-        associatedCompositeMonitors: List<AssociatedWorkflow>?,
-    ) : super() {
-        this.id = id
-        this.version = version
-        this.seqNo = seqNo
-        this.primaryTerm = primaryTerm
-        this.status = status
-        this.monitor = monitor
-        this.associatedWorkflows = associatedCompositeMonitors ?: emptyList()
-    }
-
-    @Throws(IOException::class)
-    constructor(sin: StreamInput) : this(
-        id = sin.readString(), // id
-        version = sin.readLong(), // version
-        seqNo = sin.readLong(), // seqNo
-        primaryTerm = sin.readLong(), // primaryTerm
-        status = sin.readEnum(RestStatus::class.java), // RestStatus
-        monitor = if (sin.readBoolean()) {
-            Monitor.readFrom(sin) // monitor
-        } else null,
-        associatedCompositeMonitors = sin.readList((AssociatedWorkflow)::readFrom),
-    )
-
-    @Throws(IOException::class)
-    override fun writeTo(out: StreamOutput) {
-        out.writeString(id)
-        out.writeLong(version)
-        out.writeLong(seqNo)
-        out.writeLong(primaryTerm)
-        out.writeEnum(status)
-        if (monitor != null) {
-            out.writeBoolean(true)
-            monitor?.writeTo(out)
-        } else {
-            out.writeBoolean(false)
-        }
-        associatedWorkflows?.forEach {
-            it.writeTo(out)
-        }
-    }
-
-    @Throws(IOException::class)
-    override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder {
-        builder.startObject()
-            .field(_ID, id)
-            .field(_VERSION, version)
-            .field(_SEQ_NO, seqNo)
-            .field(_PRIMARY_TERM, primaryTerm)
-        if (monitor != null) {
-            builder.field("monitor", monitor)
-        }
-        if (associatedWorkflows != null) {
-            builder.field("associated_workflows", associatedWorkflows!!.toTypedArray())
-        }
-        return builder.endObject()
-    }
-
-    class AssociatedWorkflow : ToXContentFragment {
-        val id: String
-        val name: String
-
-        constructor(id: String, name: String) {
-            this.id = id
-            this.name = name
-        }
-
-        override fun toXContent(builder: XContentBuilder, params: ToXContent.Params?): XContentBuilder {
-            builder.startObject()
-                .field("id", id)
-                .field("name", name)
-                .endObject()
-            return builder
-        }
-
-        fun writeTo(out: StreamOutput) {
-            out.writeString(id)
-            out.writeString(name)
-        }
-
-        @Throws(IOException::class)
-        constructor(sin: StreamInput) : this(
-            sin.readString(),
-            sin.readString()
-        )
-
-        companion object {
-            @JvmStatic
-            @Throws(IOException::class)
-            fun readFrom(sin: StreamInput): AssociatedWorkflow {
-                return AssociatedWorkflow(sin)
-            }
-        }
-    }
-}
diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/action/GetRemoteIndexesAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/action/GetRemoteIndexesAction.kt
new file mode 100644
index 000000000..059110af4
--- /dev/null
+++ b/alerting/src/main/kotlin/org/opensearch/alerting/action/GetRemoteIndexesAction.kt
@@ -0,0 +1,15 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.alerting.action
+
+import org.opensearch.action.ActionType
+
+class GetRemoteIndexesAction private constructor() : ActionType<GetRemoteIndexesResponse>(NAME, ::GetRemoteIndexesResponse) {
+    companion object {
+        val INSTANCE = GetRemoteIndexesAction()
+        const val NAME = "cluster:admin/opensearch/alerting/remote/indexes/get"
+    }
+}
diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/action/SearchMonitorRequest.kt b/alerting/src/main/kotlin/org/opensearch/alerting/action/GetRemoteIndexesRequest.kt
similarity index 53%
rename from alerting/src/main/kotlin/org/opensearch/alerting/action/SearchMonitorRequest.kt
rename to alerting/src/main/kotlin/org/opensearch/alerting/action/GetRemoteIndexesRequest.kt
index 069b1ed4e..733bc3a04 100644
--- a/alerting/src/main/kotlin/org/opensearch/alerting/action/SearchMonitorRequest.kt
+++ b/alerting/src/main/kotlin/org/opensearch/alerting/action/GetRemoteIndexesRequest.kt
@@ -7,24 +7,23 @@ package org.opensearch.alerting.action
 
 import org.opensearch.action.ActionRequest
 import org.opensearch.action.ActionRequestValidationException
-import org.opensearch.action.search.SearchRequest
 import org.opensearch.core.common.io.stream.StreamInput
 import org.opensearch.core.common.io.stream.StreamOutput
 import java.io.IOException
 
-class SearchMonitorRequest : ActionRequest {
+class GetRemoteIndexesRequest : ActionRequest {
+    var indexes: List<String> = listOf()
+    var includeMappings: Boolean
 
-    val searchRequest: SearchRequest
-
-    constructor(
-        searchRequest: SearchRequest
-    ) : super() {
-        this.searchRequest = searchRequest
+    constructor(indexes: List<String>, includeMappings: Boolean) : super() {
+        this.indexes = indexes
+        this.includeMappings = includeMappings
     }
 
     @Throws(IOException::class)
     constructor(sin: StreamInput) : this(
-        searchRequest = SearchRequest(sin)
+        sin.readStringList(),
+        sin.readBoolean()
     )
 
     override fun validate(): ActionRequestValidationException? {
@@ -33,6 +32,12 @@ class SearchMonitorRequest : ActionRequest {
 
     @Throws(IOException::class)
     override fun writeTo(out: StreamOutput) {
-        searchRequest.writeTo(out)
+        out.writeStringArray(indexes.toTypedArray())
+        out.writeBoolean(includeMappings)
+    }
+
+    companion object {
+        const val INDEXES_FIELD = "indexes"
+        const val INCLUDE_MAPPINGS_FIELD = "include_mappings"
     }
 }
diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/action/GetRemoteIndexesResponse.kt b/alerting/src/main/kotlin/org/opensearch/alerting/action/GetRemoteIndexesResponse.kt
new file mode 100644
index 000000000..1572b4228
--- /dev/null
+++ b/alerting/src/main/kotlin/org/opensearch/alerting/action/GetRemoteIndexesResponse.kt
@@ -0,0 +1,135 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.alerting.action
+
+import org.opensearch.cluster.health.ClusterHealthStatus
+import org.opensearch.cluster.metadata.MappingMetadata
+import org.opensearch.core.action.ActionResponse
+import org.opensearch.core.common.io.stream.StreamInput
+import org.opensearch.core.common.io.stream.StreamOutput
+import org.opensearch.core.common.io.stream.Writeable
+import org.opensearch.core.xcontent.ToXContent
+import org.opensearch.core.xcontent.ToXContentObject
+import org.opensearch.core.xcontent.XContentBuilder
+import java.io.IOException
+
+class GetRemoteIndexesResponse : ActionResponse, ToXContentObject {
+    var clusterIndexes: List<ClusterIndexes> = emptyList()
+
+    constructor(clusterIndexes: List<ClusterIndexes>) : super() {
+        this.clusterIndexes = clusterIndexes
+    }
+
+    @Throws(IOException::class)
+    constructor(sin: StreamInput) : this(
+        clusterIndexes = sin.readList((ClusterIndexes.Companion)::readFrom)
+    )
+
+    override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder {
+        builder.startObject()
+        clusterIndexes.forEach {
+            it.toXContent(builder, params)
+        }
+        return builder.endObject()
+    }
+
+    override fun writeTo(out: StreamOutput) {
+        clusterIndexes.forEach { it.writeTo(out) }
+    }
+
+    data class ClusterIndexes(
+        val clusterName: String,
+        val clusterHealth: ClusterHealthStatus,
+        val hubCluster: Boolean,
+        val indexes: List<ClusterIndex> = listOf(),
+        val latency: Long
+    ) : ToXContentObject, Writeable {
+
+        @Throws(IOException::class)
+        constructor(sin: StreamInput) : this(
+            clusterName = sin.readString(),
+            clusterHealth = sin.readEnum(ClusterHealthStatus::class.java),
+            hubCluster = sin.readBoolean(),
+            indexes = sin.readList((ClusterIndex.Companion)::readFrom),
+            latency = sin.readLong()
+        )
+
+        override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder {
+            builder.startObject(clusterName)
+            builder.field(CLUSTER_NAME_FIELD, clusterName)
+            builder.field(CLUSTER_HEALTH_FIELD, clusterHealth)
+            builder.field(HUB_CLUSTER_FIELD, hubCluster)
+            builder.field(INDEX_LATENCY_FIELD, latency)
+            builder.startObject(INDEXES_FIELD)
+            indexes.forEach {
+                it.toXContent(builder, params)
+            }
+            return builder.endObject().endObject()
+        }
+
+        override fun writeTo(out: StreamOutput) {
+            out.writeString(clusterName)
+            out.writeEnum(clusterHealth)
+            indexes.forEach { it.writeTo(out) }
+            out.writeLong(latency)
+        }
+
+        companion object {
+            const val CLUSTER_NAME_FIELD = "cluster"
+            const val CLUSTER_HEALTH_FIELD = "health"
+            const val HUB_CLUSTER_FIELD = "hub_cluster"
+            const val INDEXES_FIELD = "indexes"
+            const val INDEX_LATENCY_FIELD = "latency"
+
+            @JvmStatic
+            @Throws(IOException::class)
+            fun readFrom(sin: StreamInput): ClusterIndexes {
+                return ClusterIndexes(sin)
+            }
+        }
+
+        data class ClusterIndex(
+            val indexName: String,
+            val indexHealth: ClusterHealthStatus?,
+            val mappings: MappingMetadata?
+        ) : ToXContentObject, Writeable {
+
+            @Throws(IOException::class)
+            constructor(sin: StreamInput) : this(
+                indexName = sin.readString(),
+                indexHealth = sin.readEnum(ClusterHealthStatus::class.java),
+                mappings = sin.readOptionalWriteable(::MappingMetadata)
+            )
+
+            override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder {
+                builder.startObject(indexName)
+                builder.field(INDEX_NAME_FIELD, indexName)
+                builder.field(INDEX_HEALTH_FIELD, indexHealth)
+                if (mappings == null) builder.startObject(MAPPINGS_FIELD).endObject()
+                else builder.field(MAPPINGS_FIELD, mappings.sourceAsMap())
+                return builder.endObject()
+            }
+
+            override fun writeTo(out: StreamOutput) {
+                out.writeString(indexName)
+                out.writeEnum(indexHealth)
+                if (mappings != null) out.writeMap(mappings.sourceAsMap)
+            }
+
+            companion object {
+                const val INDEX_NAME_FIELD = "name"
+                const val INDEX_HEALTH_FIELD = "health"
+                const val MAPPINGS_FIELD = "mappings"
+
+                @JvmStatic
+                @Throws(IOException::class)
+                fun readFrom(sin: StreamInput): ClusterIndex {
+                    return ClusterIndex(sin)
+                }
+            }
+        }
+    }
+}
diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/action/SearchMonitorAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/action/SearchMonitorAction.kt
deleted file mode 100644
index 16725fc39..000000000
--- a/alerting/src/main/kotlin/org/opensearch/alerting/action/SearchMonitorAction.kt
+++ /dev/null
@@ -1,16 +0,0 @@
-/*
- * Copyright OpenSearch Contributors
- * SPDX-License-Identifier: Apache-2.0
- */
-
-package org.opensearch.alerting.action
-
-import org.opensearch.action.ActionType
-import org.opensearch.action.search.SearchResponse
-
-class SearchMonitorAction private constructor() : ActionType<SearchResponse>(NAME, ::SearchResponse) {
-    companion object {
-        val INSTANCE = SearchMonitorAction()
-        const val NAME = "cluster:admin/opendistro/alerting/monitor/search"
-    }
-}
diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/model/ClusterMetricsTriggerRunResult.kt b/alerting/src/main/kotlin/org/opensearch/alerting/model/ClusterMetricsTriggerRunResult.kt
new file mode 100644
index 000000000..a19de0637
--- /dev/null
+++ b/alerting/src/main/kotlin/org/opensearch/alerting/model/ClusterMetricsTriggerRunResult.kt
@@ -0,0 +1,110 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.alerting.model
+
+import org.opensearch.commons.alerting.alerts.AlertError
+import org.opensearch.core.common.io.stream.StreamInput
+import org.opensearch.core.common.io.stream.StreamOutput
+import org.opensearch.core.common.io.stream.Writeable
+import org.opensearch.core.xcontent.ToXContent
+import org.opensearch.core.xcontent.ToXContentObject
+import org.opensearch.core.xcontent.XContentBuilder
+import org.opensearch.script.ScriptException
+import java.io.IOException
+import java.time.Instant
+
+data class ClusterMetricsTriggerRunResult(
+    override var triggerName: String,
+    override var triggered: Boolean,
+    override var error: Exception?,
+    override var actionResults: MutableMap<String, ActionRunResult> = mutableMapOf(),
+    var clusterTriggerResults: List<ClusterTriggerResult> = listOf()
+) : QueryLevelTriggerRunResult(
+    triggerName = triggerName,
+    error = error,
+    triggered = triggered,
+    actionResults = actionResults
+) {
+
+    @Throws(IOException::class)
+    @Suppress("UNCHECKED_CAST")
+    constructor(sin: StreamInput) : this(
+        triggerName = sin.readString(),
+        error = sin.readException(),
+        triggered = sin.readBoolean(),
+        actionResults = sin.readMap() as MutableMap<String, ActionRunResult>,
+        clusterTriggerResults = sin.readList((ClusterTriggerResult.Companion)::readFrom)
+    )
+
+    override fun alertError(): AlertError? {
+        if (error != null) {
+            return AlertError(Instant.now(), "Failed evaluating trigger:\n${error!!.userErrorMessage()}")
+        }
+        for (actionResult in actionResults.values) {
+            if (actionResult.error != null) {
+                return AlertError(Instant.now(), "Failed running action:\n${actionResult.error.userErrorMessage()}")
+            }
+        }
+        return null
+    }
+
+    override fun internalXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder {
+        if (error is ScriptException) error = Exception((error as ScriptException).toJsonString(), error)
+        builder
+            .field(TRIGGERED_FIELD, triggered)
+            .field(ACTION_RESULTS_FIELD, actionResults as Map<String, ActionRunResult>)
+            .startArray(CLUSTER_RESULTS_FIELD)
+        clusterTriggerResults.forEach { it.toXContent(builder, params) }
+        return builder.endArray()
+    }
+
+    @Throws(IOException::class)
+    override fun writeTo(out: StreamOutput) {
+        super.writeTo(out)
+        out.writeBoolean(triggered)
+        out.writeMap(actionResults as Map<String, ActionRunResult>)
+        clusterTriggerResults.forEach { it.writeTo(out) }
+    }
+
+    companion object {
+        const val TRIGGERED_FIELD = "triggered"
+        const val ACTION_RESULTS_FIELD = "action_results"
+        const val CLUSTER_RESULTS_FIELD = "cluster_results"
+    }
+
+    data class ClusterTriggerResult(
+        val cluster: String,
+        val triggered: Boolean,
+    ) : ToXContentObject, Writeable {
+
+        @Throws(IOException::class)
+        constructor(sin: StreamInput) : this(
+            cluster = sin.readString(),
+            triggered = sin.readBoolean()
+        )
+
+        override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder {
+            return builder.startObject()
+                .startObject(cluster)
+                .field(TRIGGERED_FIELD, triggered)
+                .endObject()
+                .endObject()
+        }
+
+        override fun writeTo(out: StreamOutput) {
+            out.writeString(cluster)
+            out.writeBoolean(triggered)
+        }
+
+        companion object {
+            @JvmStatic
+            @Throws(IOException::class)
+            fun readFrom(sin: StreamInput): ClusterTriggerResult {
+                return ClusterTriggerResult(sin)
+            }
+        }
+    }
+}
diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/model/DocumentExecutionContext.kt b/alerting/src/main/kotlin/org/opensearch/alerting/model/DocumentExecutionContext.kt
deleted file mode 100644
index 0caad1f4a..000000000
--- a/alerting/src/main/kotlin/org/opensearch/alerting/model/DocumentExecutionContext.kt
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright OpenSearch Contributors
- * SPDX-License-Identifier: Apache-2.0
- */
-
-package org.opensearch.alerting.model
-
-import org.opensearch.commons.alerting.model.DocLevelQuery
-
-data class DocumentExecutionContext(
-    val queries: List<DocLevelQuery>,
-    val lastRunContext: Map<String, Any>,
-    val updatedLastRunContext: Map<String, Any>
-)
diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/model/IndexExecutionContext.kt b/alerting/src/main/kotlin/org/opensearch/alerting/model/IndexExecutionContext.kt
new file mode 100644
index 000000000..e7aa707f9
--- /dev/null
+++ b/alerting/src/main/kotlin/org/opensearch/alerting/model/IndexExecutionContext.kt
@@ -0,0 +1,19 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.alerting.model
+
+import org.opensearch.commons.alerting.model.DocLevelQuery
+
+/** DTO that contains all the necessary context for fetching data from shard and performing percolate queries */
+data class IndexExecutionContext(
+    val queries: List<DocLevelQuery>,
+    val lastRunContext: MutableMap<String, Any>,
+    val updatedLastRunContext: MutableMap<String, Any>,
+    val indexName: String,
+    val concreteIndexName: String,
+    val conflictingFields: List<String>,
+    val docIds: List<String>? = null,
+)
diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/model/QueryLevelTriggerRunResult.kt b/alerting/src/main/kotlin/org/opensearch/alerting/model/QueryLevelTriggerRunResult.kt
index d123dbae4..5917c1ecf 100644
--- a/alerting/src/main/kotlin/org/opensearch/alerting/model/QueryLevelTriggerRunResult.kt
+++ b/alerting/src/main/kotlin/org/opensearch/alerting/model/QueryLevelTriggerRunResult.kt
@@ -14,11 +14,11 @@ import org.opensearch.script.ScriptException
 import java.io.IOException
 import java.time.Instant
 
-data class QueryLevelTriggerRunResult(
+open class QueryLevelTriggerRunResult(
     override var triggerName: String,
-    var triggered: Boolean,
+    open var triggered: Boolean,
     override var error: Exception?,
-    var actionResults: MutableMap<String, ActionRunResult> = mutableMapOf()
+    open var actionResults: MutableMap<String, ActionRunResult> = mutableMapOf()
 ) : TriggerRunResult(triggerName, error) {
 
     @Throws(IOException::class)
diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestGetMonitorAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestGetMonitorAction.kt
index 4d9bac033..54270b717 100644
--- a/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestGetMonitorAction.kt
+++ b/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestGetMonitorAction.kt
@@ -6,10 +6,10 @@ package org.opensearch.alerting.resthandler
 
 import org.apache.logging.log4j.LogManager
 import org.opensearch.alerting.AlertingPlugin
-import org.opensearch.alerting.action.GetMonitorAction
-import org.opensearch.alerting.action.GetMonitorRequest
 import org.opensearch.alerting.util.context
 import org.opensearch.client.node.NodeClient
+import org.opensearch.commons.alerting.action.AlertingActions
+import org.opensearch.commons.alerting.action.GetMonitorRequest
 import org.opensearch.rest.BaseRestHandler
 import org.opensearch.rest.BaseRestHandler.RestChannelConsumer
 import org.opensearch.rest.RestHandler.ReplacedRoute
@@ -69,7 +69,7 @@ class RestGetMonitorAction : BaseRestHandler() {
         val getMonitorRequest = GetMonitorRequest(monitorId, RestActions.parseVersion(request), request.method(), srcContext)
         return RestChannelConsumer {
                 channel ->
-            client.execute(GetMonitorAction.INSTANCE, getMonitorRequest, RestToXContentListener(channel))
+            client.execute(AlertingActions.GET_MONITOR_ACTION_TYPE, getMonitorRequest, RestToXContentListener(channel))
         }
     }
 }
diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestGetRemoteIndexesAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestGetRemoteIndexesAction.kt
new file mode 100644
index 000000000..591ab2c3e
--- /dev/null
+++ b/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestGetRemoteIndexesAction.kt
@@ -0,0 +1,50 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.alerting.resthandler
+
+import org.apache.logging.log4j.LogManager
+import org.opensearch.alerting.AlertingPlugin
+import org.opensearch.alerting.action.GetRemoteIndexesAction
+import org.opensearch.alerting.action.GetRemoteIndexesRequest
+import org.opensearch.client.node.NodeClient
+import org.opensearch.core.common.Strings
+import org.opensearch.rest.BaseRestHandler
+import org.opensearch.rest.RestHandler
+import org.opensearch.rest.RestRequest
+import org.opensearch.rest.action.RestToXContentListener
+
+private val log = LogManager.getLogger(RestGetRemoteIndexesAction::class.java)
+
+class RestGetRemoteIndexesAction : BaseRestHandler() {
+    val ROUTE = "${AlertingPlugin.REMOTE_BASE_URI}/indexes"
+
+    override fun getName(): String {
+        return "get_remote_indexes_action"
+    }
+
+    override fun routes(): List<RestHandler.Route> {
+        return mutableListOf(
+            RestHandler.Route(RestRequest.Method.GET, ROUTE)
+        )
+    }
+
+    override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer {
+        log.debug("${request.method()} $ROUTE")
+        val indexes = Strings.splitStringByCommaToArray(request.param(GetRemoteIndexesRequest.INDEXES_FIELD, ""))
+        val includeMappings = request.paramAsBoolean(GetRemoteIndexesRequest.INCLUDE_MAPPINGS_FIELD, false)
+        return RestChannelConsumer {
+                channel ->
+            client.execute(
+                GetRemoteIndexesAction.INSTANCE,
+                GetRemoteIndexesRequest(
+                    indexes = indexes.toList(),
+                    includeMappings = includeMappings
+                ),
+                RestToXContentListener(channel)
+            )
+        }
+    }
+}
diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestSearchMonitorAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestSearchMonitorAction.kt
index 4d4ea47bf..1bf51678e 100644
--- a/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestSearchMonitorAction.kt
+++ b/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestSearchMonitorAction.kt
@@ -9,8 +9,6 @@ import org.apache.logging.log4j.LogManager
 import org.opensearch.action.search.SearchRequest
 import org.opensearch.action.search.SearchResponse
 import org.opensearch.alerting.AlertingPlugin
-import org.opensearch.alerting.action.SearchMonitorAction
-import org.opensearch.alerting.action.SearchMonitorRequest
 import org.opensearch.alerting.alerts.AlertIndices.Companion.ALL_ALERT_INDEX_PATTERN
 import org.opensearch.alerting.settings.AlertingSettings
 import org.opensearch.alerting.util.context
@@ -20,6 +18,8 @@ import org.opensearch.common.settings.Settings
 import org.opensearch.common.xcontent.LoggingDeprecationHandler
 import org.opensearch.common.xcontent.XContentFactory.jsonBuilder
 import org.opensearch.common.xcontent.XContentType
+import org.opensearch.commons.alerting.action.AlertingActions
+import org.opensearch.commons.alerting.action.SearchMonitorRequest
 import org.opensearch.commons.alerting.model.ScheduledJob
 import org.opensearch.commons.alerting.model.ScheduledJob.Companion.SCHEDULED_JOBS_INDEX
 import org.opensearch.core.common.bytes.BytesReference
@@ -101,7 +101,7 @@ class RestSearchMonitorAction(
 
         val searchMonitorRequest = SearchMonitorRequest(searchRequest)
         return RestChannelConsumer { channel ->
-            client.execute(SearchMonitorAction.INSTANCE, searchMonitorRequest, searchMonitorResponse(channel))
+            client.execute(AlertingActions.SEARCH_MONITORS_ACTION_TYPE, searchMonitorRequest, searchMonitorResponse(channel))
         }
     }
 
diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/service/DeleteMonitorService.kt b/alerting/src/main/kotlin/org/opensearch/alerting/service/DeleteMonitorService.kt
index b26ae2473..2ba1fbd17 100644
--- a/alerting/src/main/kotlin/org/opensearch/alerting/service/DeleteMonitorService.kt
+++ b/alerting/src/main/kotlin/org/opensearch/alerting/service/DeleteMonitorService.kt
@@ -22,6 +22,8 @@ import org.opensearch.action.support.IndicesOptions
 import org.opensearch.action.support.WriteRequest.RefreshPolicy
 import org.opensearch.action.support.master.AcknowledgedResponse
 import org.opensearch.alerting.MonitorMetadataService
+import org.opensearch.alerting.core.lock.LockModel
+import org.opensearch.alerting.core.lock.LockService
 import org.opensearch.alerting.opensearchapi.suspendUntil
 import org.opensearch.alerting.util.AlertingException
 import org.opensearch.alerting.util.ScheduledJobUtils.Companion.WORKFLOW_DELEGATE_PATH
@@ -48,11 +50,14 @@ object DeleteMonitorService :
     private val log = LogManager.getLogger(this.javaClass)
 
     private lateinit var client: Client
+    private lateinit var lockService: LockService
 
     fun initialize(
         client: Client,
+        lockService: LockService
     ) {
         DeleteMonitorService.client = client
+        DeleteMonitorService.lockService = lockService
     }
 
     /**
@@ -64,6 +69,7 @@ object DeleteMonitorService :
         val deleteResponse = deleteMonitor(monitor.id, refreshPolicy)
         deleteDocLevelMonitorQueriesAndIndices(monitor)
         deleteMetadata(monitor)
+        deleteLock(monitor)
         return DeleteMonitorResponse(deleteResponse.id, deleteResponse.version)
     }
 
@@ -147,6 +153,10 @@ object DeleteMonitorService :
         }
     }
 
+    private suspend fun deleteLock(monitor: Monitor) {
+        client.suspendUntil<Client, Boolean> { lockService.deleteLock(LockModel.generateLockId(monitor.id), it) }
+    }
+
     /**
      * Checks if the monitor is part of the workflow
      *
diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt
index 7dd90b106..a0eda830a 100644
--- a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt
+++ b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt
@@ -17,68 +17,106 @@ class AlertingSettings {
 
     companion object {
         const val DEFAULT_MAX_ACTIONABLE_ALERT_COUNT = 50L
+        const val DEFAULT_FINDINGS_INDEXING_BATCH_SIZE = 1000
+        const val DEFAULT_PERCOLATE_QUERY_NUM_DOCS_IN_MEMORY = 50000
+        const val DEFAULT_PERCOLATE_QUERY_DOCS_SIZE_MEMORY_PERCENTAGE_LIMIT = 10
+        const val DEFAULT_DOC_LEVEL_MONITOR_SHARD_FETCH_SIZE = 10000
 
         val ALERTING_MAX_MONITORS = Setting.intSetting(
             "plugins.alerting.monitor.max_monitors",
             LegacyOpenDistroAlertingSettings.ALERTING_MAX_MONITORS,
-            Setting.Property.NodeScope,
-            Setting.Property.Dynamic
+            Setting.Property.NodeScope, Setting.Property.Dynamic
+        )
+
+        /** Defines the threshold percentage of heap size in bytes till which we accumulate docs in memory before we query against percolate query
+         * index in document level monitor execution.
+         */
+        val PERCOLATE_QUERY_DOCS_SIZE_MEMORY_PERCENTAGE_LIMIT = Setting.intSetting(
+            "plugins.alerting.monitor.percolate_query_docs_size_memory_percentage_limit",
+            10,
+            0,
+            100,
+            Setting.Property.NodeScope, Setting.Property.Dynamic
+        )
+
+        /** Purely a setting used to verify seq_no calculation
+         */
+        val DOC_LEVEL_MONITOR_SHARD_FETCH_SIZE = Setting.intSetting(
+            "plugins.alerting.monitor.doc_level_monitor_shard_fetch_size",
+            DEFAULT_DOC_LEVEL_MONITOR_SHARD_FETCH_SIZE,
+            1,
+            10000,
+            Setting.Property.NodeScope, Setting.Property.Dynamic
+        )
+
+        /** Defines the threshold of the maximum number of docs accumulated in memory to query against percolate query index in document
+         * level monitor execution. The docs are being collected from searching on shards of indices mentioned in the
+         * monitor input indices field. When the number of in-memory docs reaches or exceeds threshold we immediately perform percolate
+         * query with the current set of docs and clear the cache and repeat the process till we have queried all indices in current
+         * execution
+         */
+        val PERCOLATE_QUERY_MAX_NUM_DOCS_IN_MEMORY = Setting.intSetting(
+            "plugins.alerting.monitor.percolate_query_max_num_docs_in_memory",
+            DEFAULT_PERCOLATE_QUERY_NUM_DOCS_IN_MEMORY, 1000,
+            Setting.Property.NodeScope, Setting.Property.Dynamic
+        )
+
+        /**
+         * Boolean setting to enable/disable optimizing doc level monitors by fetchign only fields mentioned in queries.
+         * Enabled by default. If disabled, will fetch entire source of documents while fetch data from shards.
+         */
+        val DOC_LEVEL_MONITOR_FETCH_ONLY_QUERY_FIELDS_ENABLED = Setting.boolSetting(
+            "plugins.alerting.monitor.doc_level_monitor_query_field_names_enabled",
+            true,
+            Setting.Property.NodeScope, Setting.Property.Dynamic
         )
 
         val INPUT_TIMEOUT = Setting.positiveTimeSetting(
             "plugins.alerting.input_timeout",
             LegacyOpenDistroAlertingSettings.INPUT_TIMEOUT,
-            Setting.Property.NodeScope,
-            Setting.Property.Dynamic
+            Setting.Property.NodeScope, Setting.Property.Dynamic
         )
 
         val INDEX_TIMEOUT = Setting.positiveTimeSetting(
             "plugins.alerting.index_timeout",
             LegacyOpenDistroAlertingSettings.INDEX_TIMEOUT,
-            Setting.Property.NodeScope,
-            Setting.Property.Dynamic
+            Setting.Property.NodeScope, Setting.Property.Dynamic
         )
 
         val BULK_TIMEOUT = Setting.positiveTimeSetting(
             "plugins.alerting.bulk_timeout",
             LegacyOpenDistroAlertingSettings.BULK_TIMEOUT,
-            Setting.Property.NodeScope,
-            Setting.Property.Dynamic
+            Setting.Property.NodeScope, Setting.Property.Dynamic
         )
 
         val ALERT_BACKOFF_MILLIS = Setting.positiveTimeSetting(
             "plugins.alerting.alert_backoff_millis",
             LegacyOpenDistroAlertingSettings.ALERT_BACKOFF_MILLIS,
-            Setting.Property.NodeScope,
-            Setting.Property.Dynamic
+            Setting.Property.NodeScope, Setting.Property.Dynamic
         )
 
         val ALERT_BACKOFF_COUNT = Setting.intSetting(
             "plugins.alerting.alert_backoff_count",
             LegacyOpenDistroAlertingSettings.ALERT_BACKOFF_COUNT,
-            Setting.Property.NodeScope,
-            Setting.Property.Dynamic
+            Setting.Property.NodeScope, Setting.Property.Dynamic
         )
 
         val MOVE_ALERTS_BACKOFF_MILLIS = Setting.positiveTimeSetting(
             "plugins.alerting.move_alerts_backoff_millis",
             LegacyOpenDistroAlertingSettings.MOVE_ALERTS_BACKOFF_MILLIS,
-            Setting.Property.NodeScope,
-            Setting.Property.Dynamic
+            Setting.Property.NodeScope, Setting.Property.Dynamic
         )
 
         val MOVE_ALERTS_BACKOFF_COUNT = Setting.intSetting(
             "plugins.alerting.move_alerts_backoff_count",
             LegacyOpenDistroAlertingSettings.MOVE_ALERTS_BACKOFF_COUNT,
-            Setting.Property.NodeScope,
-            Setting.Property.Dynamic
+            Setting.Property.NodeScope, Setting.Property.Dynamic
         )
 
         val ALERT_HISTORY_ENABLED = Setting.boolSetting(
             "plugins.alerting.alert_history_enabled",
             LegacyOpenDistroAlertingSettings.ALERT_HISTORY_ENABLED,
-            Setting.Property.NodeScope,
-            Setting.Property.Dynamic
+            Setting.Property.NodeScope, Setting.Property.Dynamic
         )
 
         // TODO: Do we want to let users to disable this? If so, we need to fix the rollover logic
@@ -86,95 +124,94 @@ class AlertingSettings {
         val FINDING_HISTORY_ENABLED = Setting.boolSetting(
             "plugins.alerting.alert_finding_enabled",
             true,
-            Setting.Property.NodeScope,
-            Setting.Property.Dynamic
+            Setting.Property.NodeScope, Setting.Property.Dynamic
         )
 
         val ALERT_HISTORY_ROLLOVER_PERIOD = Setting.positiveTimeSetting(
             "plugins.alerting.alert_history_rollover_period",
             LegacyOpenDistroAlertingSettings.ALERT_HISTORY_ROLLOVER_PERIOD,
-            Setting.Property.NodeScope,
-            Setting.Property.Dynamic
+            Setting.Property.NodeScope, Setting.Property.Dynamic
         )
 
         val FINDING_HISTORY_ROLLOVER_PERIOD = Setting.positiveTimeSetting(
             "plugins.alerting.alert_finding_rollover_period",
             TimeValue.timeValueHours(12),
-            Setting.Property.NodeScope,
-            Setting.Property.Dynamic
+            Setting.Property.NodeScope, Setting.Property.Dynamic
         )
 
         val ALERT_HISTORY_INDEX_MAX_AGE = Setting.positiveTimeSetting(
             "plugins.alerting.alert_history_max_age",
             LegacyOpenDistroAlertingSettings.ALERT_HISTORY_INDEX_MAX_AGE,
-            Setting.Property.NodeScope,
-            Setting.Property.Dynamic
+            Setting.Property.NodeScope, Setting.Property.Dynamic
         )
 
         val FINDING_HISTORY_INDEX_MAX_AGE = Setting.positiveTimeSetting(
             "plugins.alerting.finding_history_max_age",
             TimeValue(30, TimeUnit.DAYS),
-            Setting.Property.NodeScope,
-            Setting.Property.Dynamic
+            Setting.Property.NodeScope, Setting.Property.Dynamic
         )
 
         val ALERT_HISTORY_MAX_DOCS = Setting.longSetting(
             "plugins.alerting.alert_history_max_docs",
             LegacyOpenDistroAlertingSettings.ALERT_HISTORY_MAX_DOCS,
-            Setting.Property.NodeScope,
-            Setting.Property.Dynamic
+            Setting.Property.NodeScope, Setting.Property.Dynamic
         )
 
         val FINDING_HISTORY_MAX_DOCS = Setting.longSetting(
             "plugins.alerting.alert_finding_max_docs",
             1000L,
             0L,
-            Setting.Property.NodeScope,
-            Setting.Property.Dynamic,
-            Setting.Property.Deprecated
+            Setting.Property.NodeScope, Setting.Property.Dynamic, Setting.Property.Deprecated
         )
 
         val ALERT_HISTORY_RETENTION_PERIOD = Setting.positiveTimeSetting(
             "plugins.alerting.alert_history_retention_period",
             LegacyOpenDistroAlertingSettings.ALERT_HISTORY_RETENTION_PERIOD,
-            Setting.Property.NodeScope,
-            Setting.Property.Dynamic
+            Setting.Property.NodeScope, Setting.Property.Dynamic
         )
 
         val FINDING_HISTORY_RETENTION_PERIOD = Setting.positiveTimeSetting(
             "plugins.alerting.finding_history_retention_period",
             TimeValue(60, TimeUnit.DAYS),
-            Setting.Property.NodeScope,
-            Setting.Property.Dynamic
+            Setting.Property.NodeScope, Setting.Property.Dynamic
         )
 
         val REQUEST_TIMEOUT = Setting.positiveTimeSetting(
             "plugins.alerting.request_timeout",
             LegacyOpenDistroAlertingSettings.REQUEST_TIMEOUT,
-            Setting.Property.NodeScope,
-            Setting.Property.Dynamic
+            Setting.Property.NodeScope, Setting.Property.Dynamic
         )
 
         val MAX_ACTION_THROTTLE_VALUE = Setting.positiveTimeSetting(
             "plugins.alerting.action_throttle_max_value",
             LegacyOpenDistroAlertingSettings.MAX_ACTION_THROTTLE_VALUE,
-            Setting.Property.NodeScope,
-            Setting.Property.Dynamic
+            Setting.Property.NodeScope, Setting.Property.Dynamic
         )
 
         val FILTER_BY_BACKEND_ROLES = Setting.boolSetting(
             "plugins.alerting.filter_by_backend_roles",
             LegacyOpenDistroAlertingSettings.FILTER_BY_BACKEND_ROLES,
-            Setting.Property.NodeScope,
-            Setting.Property.Dynamic
+            Setting.Property.NodeScope, Setting.Property.Dynamic
         )
 
         val MAX_ACTIONABLE_ALERT_COUNT = Setting.longSetting(
             "plugins.alerting.max_actionable_alert_count",
             DEFAULT_MAX_ACTIONABLE_ALERT_COUNT,
             -1L,
-            Setting.Property.NodeScope,
-            Setting.Property.Dynamic
+            Setting.Property.NodeScope, Setting.Property.Dynamic
+        )
+
+        val FINDINGS_INDEXING_BATCH_SIZE = Setting.intSetting(
+            "plugins.alerting.alert_findings_indexing_batch_size",
+            DEFAULT_FINDINGS_INDEXING_BATCH_SIZE,
+            1,
+            Setting.Property.NodeScope, Setting.Property.Dynamic
+        )
+
+        val REMOTE_MONITORING_ENABLED = Setting.boolSetting(
+            "plugins.alerting.remote_monitoring_enabled",
+            false,
+            Setting.Property.NodeScope, Setting.Property.Dynamic
         )
     }
 }
diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportAcknowledgeAlertAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportAcknowledgeAlertAction.kt
index f76005193..a94a682d3 100644
--- a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportAcknowledgeAlertAction.kt
+++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportAcknowledgeAlertAction.kt
@@ -20,9 +20,6 @@ import org.opensearch.action.search.SearchResponse
 import org.opensearch.action.support.ActionFilters
 import org.opensearch.action.support.HandledTransportAction
 import org.opensearch.action.update.UpdateRequest
-import org.opensearch.alerting.action.GetMonitorAction
-import org.opensearch.alerting.action.GetMonitorRequest
-import org.opensearch.alerting.action.GetMonitorResponse
 import org.opensearch.alerting.opensearchapi.suspendUntil
 import org.opensearch.alerting.settings.AlertingSettings
 import org.opensearch.alerting.util.AlertingException
@@ -37,6 +34,8 @@ import org.opensearch.common.xcontent.XContentType
 import org.opensearch.commons.alerting.action.AcknowledgeAlertRequest
 import org.opensearch.commons.alerting.action.AcknowledgeAlertResponse
 import org.opensearch.commons.alerting.action.AlertingActions
+import org.opensearch.commons.alerting.action.GetMonitorRequest
+import org.opensearch.commons.alerting.action.GetMonitorResponse
 import org.opensearch.commons.alerting.model.Alert
 import org.opensearch.commons.alerting.model.Monitor
 import org.opensearch.commons.alerting.util.optionalTimeField
@@ -96,7 +95,7 @@ class TransportAcknowledgeAlertAction @Inject constructor(
                             RestRequest.Method.GET,
                             FetchSourceContext.FETCH_SOURCE
                         )
-                        execute(GetMonitorAction.INSTANCE, getMonitorRequest, it)
+                        execute(AlertingActions.GET_MONITOR_ACTION_TYPE, getMonitorRequest, it)
                     }
                 if (getMonitorResponse.monitor == null) {
                     actionListener.onFailure(
diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportDeleteWorkflowAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportDeleteWorkflowAction.kt
index 9b076a600..f5c062c88 100644
--- a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportDeleteWorkflowAction.kt
+++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportDeleteWorkflowAction.kt
@@ -23,6 +23,8 @@ import org.opensearch.action.search.SearchResponse
 import org.opensearch.action.support.ActionFilters
 import org.opensearch.action.support.HandledTransportAction
 import org.opensearch.action.support.WriteRequest.RefreshPolicy
+import org.opensearch.alerting.core.lock.LockModel
+import org.opensearch.alerting.core.lock.LockService
 import org.opensearch.alerting.model.MonitorMetadata
 import org.opensearch.alerting.model.WorkflowMetadata
 import org.opensearch.alerting.opensearchapi.addFilter
@@ -73,6 +75,7 @@ class TransportDeleteWorkflowAction @Inject constructor(
     val clusterService: ClusterService,
     val settings: Settings,
     val xContentRegistry: NamedXContentRegistry,
+    val lockService: LockService
 ) : HandledTransportAction<ActionRequest, DeleteWorkflowResponse>(
     AlertingActions.DELETE_WORKFLOW_ACTION_NAME, transportService, actionFilters, ::DeleteWorkflowRequest
 ),
@@ -119,7 +122,7 @@ class TransportDeleteWorkflowAction @Inject constructor(
     ) {
         suspend fun resolveUserAndStart() {
             try {
-                val workflow = getWorkflow()
+                val workflow: Workflow = getWorkflow() ?: return
 
                 val canDelete = user == null ||
                     !doFilterForUser(user) ||
@@ -180,6 +183,12 @@ class TransportDeleteWorkflowAction @Inject constructor(
                     } catch (t: Exception) {
                         log.error("Failed to delete delegate monitor metadata. But proceeding with workflow deletion $workflowId", t)
                     }
+                    try {
+                        // Delete the workflow lock
+                        client.suspendUntil<Client, Boolean> { lockService.deleteLock(LockModel.generateLockId(workflowId), it) }
+                    } catch (t: Exception) {
+                        log.error("Failed to delete workflow lock for $workflowId")
+                    }
                     actionListener.onResponse(deleteWorkflowResponse)
                 } else {
                     actionListener.onFailure(
@@ -296,17 +305,27 @@ class TransportDeleteWorkflowAction @Inject constructor(
             return deletableMonitors
         }
 
-        private suspend fun getWorkflow(): Workflow {
+        private suspend fun getWorkflow(): Workflow? {
             val getRequest = GetRequest(ScheduledJob.SCHEDULED_JOBS_INDEX, workflowId)
 
             val getResponse: GetResponse = client.suspendUntil { get(getRequest, it) }
-            if (getResponse.isExists == false) {
-                actionListener.onFailure(
-                    AlertingException.wrap(
-                        OpenSearchStatusException("Workflow not found.", RestStatus.NOT_FOUND)
-                    )
-                )
+            if (!getResponse.isExists) {
+                handleWorkflowMissing()
+                return null
             }
+
+            return parseWorkflow(getResponse)
+        }
+
+        private fun handleWorkflowMissing() {
+            actionListener.onFailure(
+                AlertingException.wrap(
+                    OpenSearchStatusException("Workflow not found.", RestStatus.NOT_FOUND)
+                )
+            )
+        }
+
+        private fun parseWorkflow(getResponse: GetResponse): Workflow {
             val xcp = XContentHelper.createParser(
                 xContentRegistry, LoggingDeprecationHandler.INSTANCE,
                 getResponse.sourceAsBytesRef, XContentType.JSON
diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportGetFindingsAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportGetFindingsAction.kt
index 302a9f22e..d5d5cc9d9 100644
--- a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportGetFindingsAction.kt
+++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportGetFindingsAction.kt
@@ -18,9 +18,6 @@ import org.opensearch.action.search.SearchRequest
 import org.opensearch.action.search.SearchResponse
 import org.opensearch.action.support.ActionFilters
 import org.opensearch.action.support.HandledTransportAction
-import org.opensearch.alerting.action.GetMonitorAction
-import org.opensearch.alerting.action.GetMonitorRequest
-import org.opensearch.alerting.action.GetMonitorResponse
 import org.opensearch.alerting.alerts.AlertIndices.Companion.ALL_FINDING_INDEX_PATTERN
 import org.opensearch.alerting.opensearchapi.suspendUntil
 import org.opensearch.alerting.settings.AlertingSettings
@@ -34,6 +31,8 @@ import org.opensearch.common.xcontent.XContentType
 import org.opensearch.commons.alerting.action.AlertingActions
 import org.opensearch.commons.alerting.action.GetFindingsRequest
 import org.opensearch.commons.alerting.action.GetFindingsResponse
+import org.opensearch.commons.alerting.action.GetMonitorRequest
+import org.opensearch.commons.alerting.action.GetMonitorResponse
 import org.opensearch.commons.alerting.model.Finding
 import org.opensearch.commons.alerting.model.FindingDocument
 import org.opensearch.commons.alerting.model.FindingWithDocs
@@ -170,7 +169,7 @@ class TransportGetFindingsSearchAction @Inject constructor(
                 )
                 val getMonitorResponse: GetMonitorResponse =
                     this@TransportGetFindingsSearchAction.client.suspendUntil {
-                        execute(GetMonitorAction.INSTANCE, getMonitorRequest, it)
+                        execute(AlertingActions.GET_MONITOR_ACTION_TYPE, getMonitorRequest, it)
                     }
                 indexName = getMonitorResponse.monitor?.dataSources?.findingsIndex ?: ALL_FINDING_INDEX_PATTERN
             }
@@ -221,8 +220,9 @@ class TransportGetFindingsSearchAction @Inject constructor(
         val documents: MutableMap<String, FindingDocument> = mutableMapOf()
         response.responses.forEach {
             val key = "${it.index}|${it.id}"
-            val docData = if (it.isFailed) "" else it.response.sourceAsString
-            val findingDocument = FindingDocument(it.index, it.id, !it.isFailed, docData)
+            val isDocFound = !(it.isFailed || it.response.sourceAsString == null)
+            val docData = if (isDocFound) it.response.sourceAsString else ""
+            val findingDocument = FindingDocument(it.index, it.id, isDocFound, docData)
             documents[key] = findingDocument
         }
 
diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportGetMonitorAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportGetMonitorAction.kt
index 0db73db2d..3a6f090ec 100644
--- a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportGetMonitorAction.kt
+++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportGetMonitorAction.kt
@@ -11,16 +11,13 @@ import kotlinx.coroutines.launch
 import org.apache.logging.log4j.LogManager
 import org.apache.lucene.search.join.ScoreMode
 import org.opensearch.OpenSearchStatusException
+import org.opensearch.action.ActionRequest
 import org.opensearch.action.get.GetRequest
 import org.opensearch.action.get.GetResponse
 import org.opensearch.action.search.SearchRequest
 import org.opensearch.action.search.SearchResponse
 import org.opensearch.action.support.ActionFilters
 import org.opensearch.action.support.HandledTransportAction
-import org.opensearch.alerting.action.GetMonitorAction
-import org.opensearch.alerting.action.GetMonitorRequest
-import org.opensearch.alerting.action.GetMonitorResponse
-import org.opensearch.alerting.action.GetMonitorResponse.AssociatedWorkflow
 import org.opensearch.alerting.opensearchapi.suspendUntil
 import org.opensearch.alerting.settings.AlertingSettings
 import org.opensearch.alerting.util.AlertingException
@@ -33,9 +30,14 @@ import org.opensearch.common.settings.Settings
 import org.opensearch.common.xcontent.LoggingDeprecationHandler
 import org.opensearch.common.xcontent.XContentHelper
 import org.opensearch.common.xcontent.XContentType
+import org.opensearch.commons.alerting.action.AlertingActions
+import org.opensearch.commons.alerting.action.GetMonitorRequest
+import org.opensearch.commons.alerting.action.GetMonitorResponse
+import org.opensearch.commons.alerting.action.GetMonitorResponse.AssociatedWorkflow
 import org.opensearch.commons.alerting.model.Monitor
 import org.opensearch.commons.alerting.model.ScheduledJob
 import org.opensearch.commons.alerting.model.Workflow
+import org.opensearch.commons.utils.recreateObject
 import org.opensearch.core.action.ActionListener
 import org.opensearch.core.rest.RestStatus
 import org.opensearch.core.xcontent.NamedXContentRegistry
@@ -54,8 +56,8 @@ class TransportGetMonitorAction @Inject constructor(
     val xContentRegistry: NamedXContentRegistry,
     val clusterService: ClusterService,
     settings: Settings,
-) : HandledTransportAction<GetMonitorRequest, GetMonitorResponse>(
-    GetMonitorAction.NAME,
+) : HandledTransportAction<ActionRequest, GetMonitorResponse>(
+    AlertingActions.GET_MONITOR_ACTION_NAME,
     transportService,
     actionFilters,
     ::GetMonitorRequest
@@ -69,12 +71,17 @@ class TransportGetMonitorAction @Inject constructor(
         listenFilterBySettingChange(clusterService)
     }
 
-    override fun doExecute(task: Task, getMonitorRequest: GetMonitorRequest, actionListener: ActionListener<GetMonitorResponse>) {
+    override fun doExecute(task: Task, request: ActionRequest, actionListener: ActionListener<GetMonitorResponse>) {
+        val transformedRequest = request as? GetMonitorRequest
+            ?: recreateObject(request) {
+                GetMonitorRequest(it)
+            }
+
         val user = readUserFromThreadContext(client)
 
-        val getRequest = GetRequest(ScheduledJob.SCHEDULED_JOBS_INDEX, getMonitorRequest.monitorId)
-            .version(getMonitorRequest.version)
-            .fetchSourceContext(getMonitorRequest.srcContext)
+        val getRequest = GetRequest(ScheduledJob.SCHEDULED_JOBS_INDEX, transformedRequest.monitorId)
+            .version(transformedRequest.version)
+            .fetchSourceContext(transformedRequest.srcContext)
 
         if (!validateUserBackendRoles(user, actionListener)) {
             return
@@ -114,7 +121,7 @@ class TransportGetMonitorAction @Inject constructor(
                                         monitor?.user,
                                         actionListener,
                                         "monitor",
-                                        getMonitorRequest.monitorId
+                                        transformedRequest.monitorId
                                     )
                                 ) {
                                     return
@@ -130,7 +137,6 @@ class TransportGetMonitorAction @Inject constructor(
                                         response.version,
                                         response.seqNo,
                                         response.primaryTerm,
-                                        RestStatus.OK,
                                         monitor,
                                         associatedCompositeMonitors
                                     )
diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportGetRemoteIndexesAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportGetRemoteIndexesAction.kt
new file mode 100644
index 000000000..5b35d493a
--- /dev/null
+++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportGetRemoteIndexesAction.kt
@@ -0,0 +1,193 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.alerting.transport
+
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.newSingleThreadContext
+import kotlinx.coroutines.withContext
+import org.apache.logging.log4j.LogManager
+import org.opensearch.OpenSearchStatusException
+import org.opensearch.action.admin.cluster.health.ClusterHealthRequest
+import org.opensearch.action.admin.cluster.health.ClusterHealthResponse
+import org.opensearch.action.admin.indices.mapping.get.GetMappingsRequest
+import org.opensearch.action.admin.indices.mapping.get.GetMappingsResponse
+import org.opensearch.action.admin.indices.resolve.ResolveIndexAction
+import org.opensearch.action.support.ActionFilters
+import org.opensearch.action.support.HandledTransportAction
+import org.opensearch.action.support.IndicesOptions
+import org.opensearch.alerting.action.GetRemoteIndexesAction
+import org.opensearch.alerting.action.GetRemoteIndexesRequest
+import org.opensearch.alerting.action.GetRemoteIndexesResponse
+import org.opensearch.alerting.action.GetRemoteIndexesResponse.ClusterIndexes
+import org.opensearch.alerting.action.GetRemoteIndexesResponse.ClusterIndexes.ClusterIndex
+import org.opensearch.alerting.opensearchapi.suspendUntil
+import org.opensearch.alerting.settings.AlertingSettings
+import org.opensearch.alerting.settings.AlertingSettings.Companion.REMOTE_MONITORING_ENABLED
+import org.opensearch.alerting.util.AlertingException
+import org.opensearch.alerting.util.CrossClusterMonitorUtils
+import org.opensearch.client.Client
+import org.opensearch.cluster.service.ClusterService
+import org.opensearch.common.inject.Inject
+import org.opensearch.common.settings.Settings
+import org.opensearch.core.action.ActionListener
+import org.opensearch.core.rest.RestStatus
+import org.opensearch.core.xcontent.NamedXContentRegistry
+import org.opensearch.tasks.Task
+import org.opensearch.transport.TransportService
+import java.time.Duration
+import java.time.Instant
+
+private val log = LogManager.getLogger(TransportGetRemoteIndexesAction::class.java)
+private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO)
+
+class TransportGetRemoteIndexesAction @Inject constructor(
+    val transportService: TransportService,
+    val client: Client,
+    actionFilters: ActionFilters,
+    val xContentRegistry: NamedXContentRegistry,
+    val clusterService: ClusterService,
+    settings: Settings,
+) : HandledTransportAction<GetRemoteIndexesRequest, GetRemoteIndexesResponse>(
+    GetRemoteIndexesAction.NAME,
+    transportService,
+    actionFilters,
+    ::GetRemoteIndexesRequest
+),
+    SecureTransportAction {
+
+    @Volatile override var filterByEnabled = AlertingSettings.FILTER_BY_BACKEND_ROLES.get(settings)
+
+    @Volatile private var remoteMonitoringEnabled = REMOTE_MONITORING_ENABLED.get(settings)
+
+    init {
+        clusterService.clusterSettings.addSettingsUpdateConsumer(REMOTE_MONITORING_ENABLED) { remoteMonitoringEnabled = it }
+        listenFilterBySettingChange(clusterService)
+    }
+
+    override fun doExecute(
+        task: Task,
+        request: GetRemoteIndexesRequest,
+        actionListener: ActionListener<GetRemoteIndexesResponse>
+    ) {
+        log.debug("Remote monitoring enabled: {}", remoteMonitoringEnabled)
+        if (!remoteMonitoringEnabled) {
+            actionListener.onFailure(
+                AlertingException.wrap(
+                    OpenSearchStatusException("Remote monitoring is not enabled.", RestStatus.FORBIDDEN)
+                )
+            )
+            return
+        }
+
+        val user = readUserFromThreadContext(client)
+        if (!validateUserBackendRoles(user, actionListener)) return
+
+        client.threadPool().threadContext.stashContext().use {
+            scope.launch {
+                val singleThreadContext = newSingleThreadContext("GetRemoteIndexesActionThread")
+                withContext(singleThreadContext) {
+                    it.restore()
+                    val clusterIndexesList = mutableListOf<ClusterIndexes>()
+
+                    var resolveIndexResponse: ResolveIndexAction.Response? = null
+                    try {
+                        resolveIndexResponse =
+                            getRemoteClusters(CrossClusterMonitorUtils.parseIndexesForRemoteSearch(request.indexes, clusterService))
+                    } catch (e: Exception) {
+                        log.error("Failed to retrieve indexes for request $request", e)
+                        actionListener.onFailure(AlertingException.wrap(e))
+                    }
+
+                    val resolvedIndexes: MutableList<String> = mutableListOf()
+                    if (resolveIndexResponse != null) {
+                        resolveIndexResponse.indices.forEach { resolvedIndexes.add(it.name) }
+                        resolveIndexResponse.aliases.forEach { resolvedIndexes.add(it.name) }
+                    }
+
+                    val clusterIndexesMap = CrossClusterMonitorUtils.separateClusterIndexes(resolvedIndexes, clusterService)
+
+                    clusterIndexesMap.forEach { (clusterName, indexes) ->
+                        val targetClient = CrossClusterMonitorUtils.getClientForCluster(clusterName, client, clusterService)
+
+                        val startTime = Instant.now()
+                        var clusterHealthResponse: ClusterHealthResponse? = null
+                        try {
+                            clusterHealthResponse = getHealthStatuses(targetClient, indexes)
+                        } catch (e: Exception) {
+                            log.error("Failed to retrieve health statuses for request $request", e)
+                            actionListener.onFailure(AlertingException.wrap(e))
+                        }
+                        val endTime = Instant.now()
+                        val latency = Duration.between(startTime, endTime).toMillis()
+
+                        var mappingsResponse: GetMappingsResponse? = null
+                        if (request.includeMappings) {
+                            try {
+                                mappingsResponse = getIndexMappings(targetClient, indexes)
+                            } catch (e: Exception) {
+                                log.error("Failed to retrieve mappings for request $request", e)
+                                actionListener.onFailure(AlertingException.wrap(e))
+                            }
+                        }
+
+                        val clusterIndexList = mutableListOf<ClusterIndex>()
+                        if (clusterHealthResponse != null) {
+                            indexes.forEach {
+                                clusterIndexList.add(
+                                    ClusterIndex(
+                                        indexName = it,
+                                        indexHealth = clusterHealthResponse.indices[it]?.status,
+                                        mappings = mappingsResponse?.mappings?.get(it)
+                                    )
+                                )
+                            }
+                        }
+
+                        clusterIndexesList.add(
+                            ClusterIndexes(
+                                clusterName = clusterName,
+                                clusterHealth = clusterHealthResponse!!.status,
+                                hubCluster = clusterName == clusterService.clusterName.value(),
+                                indexes = clusterIndexList,
+                                latency = latency
+                            )
+                        )
+                    }
+                    actionListener.onResponse(GetRemoteIndexesResponse(clusterIndexes = clusterIndexesList))
+                }
+            }
+        }
+    }
+
+    private suspend fun getRemoteClusters(parsedIndexes: List<String>): ResolveIndexAction.Response {
+        val resolveRequest = ResolveIndexAction.Request(
+            parsedIndexes.toTypedArray(),
+            ResolveIndexAction.Request.DEFAULT_INDICES_OPTIONS
+        )
+
+        return client.suspendUntil {
+            admin().indices().resolveIndex(resolveRequest, it)
+        }
+    }
+    private suspend fun getHealthStatuses(targetClient: Client, parsedIndexesNames: List<String>): ClusterHealthResponse {
+        val clusterHealthRequest = ClusterHealthRequest()
+            .indices(*parsedIndexesNames.toTypedArray())
+            .indicesOptions(IndicesOptions.lenientExpandHidden())
+
+        return targetClient.suspendUntil {
+            admin().cluster().health(clusterHealthRequest, it)
+        }
+    }
+
+    private suspend fun getIndexMappings(targetClient: Client, parsedIndexNames: List<String>): GetMappingsResponse {
+        val getMappingsRequest = GetMappingsRequest().indices(*parsedIndexNames.toTypedArray())
+        return targetClient.suspendUntil {
+            admin().indices().getMappings(getMappingsRequest, it)
+        }
+    }
+}
diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorAction.kt
index 2100c0593..057977079 100644
--- a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorAction.kt
+++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportIndexMonitorAction.kt
@@ -198,7 +198,15 @@ class TransportIndexMonitorAction @Inject constructor(
             else (it as DocLevelMonitorInput).indices
             indices.addAll(inputIndices)
         }
-        val searchRequest = SearchRequest().indices(*indices.toTypedArray())
+        val updatedIndices = indices.map { index ->
+            if (IndexUtils.isAlias(index, clusterService.state()) || IndexUtils.isDataStream(index, clusterService.state())) {
+                val metadata = clusterService.state().metadata.indicesLookup[index]?.writeIndex
+                metadata?.index?.name ?: index
+            } else {
+                index
+            }
+        }
+        val searchRequest = SearchRequest().indices(*updatedIndices.toTypedArray())
             .source(SearchSourceBuilder.searchSource().size(1).query(QueryBuilders.matchAllQuery()))
         client.search(
             searchRequest,
diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportSearchMonitorAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportSearchMonitorAction.kt
index d1b533282..7359d60ea 100644
--- a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportSearchMonitorAction.kt
+++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportSearchMonitorAction.kt
@@ -6,12 +6,11 @@
 package org.opensearch.alerting.transport
 
 import org.apache.logging.log4j.LogManager
+import org.opensearch.action.ActionRequest
 import org.opensearch.action.search.SearchRequest
 import org.opensearch.action.search.SearchResponse
 import org.opensearch.action.support.ActionFilters
 import org.opensearch.action.support.HandledTransportAction
-import org.opensearch.alerting.action.SearchMonitorAction
-import org.opensearch.alerting.action.SearchMonitorRequest
 import org.opensearch.alerting.opensearchapi.addFilter
 import org.opensearch.alerting.settings.AlertingSettings
 import org.opensearch.alerting.util.AlertingException
@@ -19,11 +18,15 @@ import org.opensearch.client.Client
 import org.opensearch.cluster.service.ClusterService
 import org.opensearch.common.inject.Inject
 import org.opensearch.common.settings.Settings
+import org.opensearch.commons.alerting.action.AlertingActions
+import org.opensearch.commons.alerting.action.SearchMonitorRequest
 import org.opensearch.commons.alerting.model.Monitor
 import org.opensearch.commons.alerting.model.ScheduledJob
 import org.opensearch.commons.alerting.model.Workflow
 import org.opensearch.commons.authuser.User
+import org.opensearch.commons.utils.recreateObject
 import org.opensearch.core.action.ActionListener
+import org.opensearch.core.common.io.stream.NamedWriteableRegistry
 import org.opensearch.index.query.BoolQueryBuilder
 import org.opensearch.index.query.ExistsQueryBuilder
 import org.opensearch.index.query.MatchQueryBuilder
@@ -38,12 +41,10 @@ class TransportSearchMonitorAction @Inject constructor(
     val settings: Settings,
     val client: Client,
     clusterService: ClusterService,
-    actionFilters: ActionFilters
-) : HandledTransportAction<SearchMonitorRequest, SearchResponse>(
-    SearchMonitorAction.NAME,
-    transportService,
-    actionFilters,
-    ::SearchMonitorRequest
+    actionFilters: ActionFilters,
+    val namedWriteableRegistry: NamedWriteableRegistry
+) : HandledTransportAction<ActionRequest, SearchResponse>(
+    AlertingActions.SEARCH_MONITORS_ACTION_NAME, transportService, actionFilters, ::SearchMonitorRequest
 ),
     SecureTransportAction {
     @Volatile
@@ -52,8 +53,13 @@ class TransportSearchMonitorAction @Inject constructor(
         listenFilterBySettingChange(clusterService)
     }
 
-    override fun doExecute(task: Task, searchMonitorRequest: SearchMonitorRequest, actionListener: ActionListener<SearchResponse>) {
-        val searchSourceBuilder = searchMonitorRequest.searchRequest.source()
+    override fun doExecute(task: Task, request: ActionRequest, actionListener: ActionListener<SearchResponse>) {
+        val transformedRequest = request as? SearchMonitorRequest
+            ?: recreateObject(request, namedWriteableRegistry) {
+                SearchMonitorRequest(it)
+            }
+
+        val searchSourceBuilder = transformedRequest.searchRequest.source()
             .seqNoAndPrimaryTerm(true)
             .version(true)
         val queryBuilder = if (searchSourceBuilder.query() == null) BoolQueryBuilder()
@@ -62,7 +68,7 @@ class TransportSearchMonitorAction @Inject constructor(
         // The SearchMonitor API supports one 'index' parameter of either the SCHEDULED_JOBS_INDEX or ALL_ALERT_INDEX_PATTERN.
         // When querying the ALL_ALERT_INDEX_PATTERN, we don't want to check whether the MONITOR_TYPE field exists
         // because we're querying alert indexes.
-        if (searchMonitorRequest.searchRequest.indices().contains(ScheduledJob.SCHEDULED_JOBS_INDEX)) {
+        if (transformedRequest.searchRequest.indices().contains(ScheduledJob.SCHEDULED_JOBS_INDEX)) {
             val monitorWorkflowType = QueryBuilders.boolQuery().should(QueryBuilders.existsQuery(Monitor.MONITOR_TYPE))
                 .should(QueryBuilders.existsQuery(Workflow.WORKFLOW_TYPE))
             queryBuilder.must(monitorWorkflowType)
@@ -71,16 +77,16 @@ class TransportSearchMonitorAction @Inject constructor(
         searchSourceBuilder.query(queryBuilder)
             .seqNoAndPrimaryTerm(true)
             .version(true)
-        addOwnerFieldIfNotExists(searchMonitorRequest.searchRequest)
+        addOwnerFieldIfNotExists(transformedRequest.searchRequest)
         val user = readUserFromThreadContext(client)
         client.threadPool().threadContext.stashContext().use {
-            resolve(searchMonitorRequest, actionListener, user)
+            resolve(transformedRequest, actionListener, user)
         }
     }
 
     fun resolve(searchMonitorRequest: SearchMonitorRequest, actionListener: ActionListener<SearchResponse>, user: User?) {
         if (user == null) {
-            // user header is null when: 1/ security is disabled. 2/when user is super-admin.
+            // user header is null when: 1/ security is disabled. 2/ when user is super-admin.
             search(searchMonitorRequest.searchRequest, actionListener)
         } else if (!doFilterForUser(user)) {
             // security is enabled and filterby is disabled.
diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/CrossClusterMonitorUtils.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/CrossClusterMonitorUtils.kt
new file mode 100644
index 000000000..6ec14ffa2
--- /dev/null
+++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/CrossClusterMonitorUtils.kt
@@ -0,0 +1,231 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.alerting.util
+
+import org.opensearch.action.search.SearchRequest
+import org.opensearch.client.Client
+import org.opensearch.client.node.NodeClient
+import org.opensearch.cluster.service.ClusterService
+import org.opensearch.commons.alerting.model.ClusterMetricsInput
+import org.opensearch.commons.alerting.model.DocLevelMonitorInput
+import org.opensearch.commons.alerting.model.Monitor
+import org.opensearch.commons.alerting.model.SearchInput
+
+class CrossClusterMonitorUtils {
+    companion object {
+
+        /**
+         * Uses the monitor inputs to determine whether the monitor makes calls to remote clusters.
+         * @param monitor The monitor to evaluate.
+         * @param localClusterName The name of the local cluster.
+         * @return TRUE if the monitor makes calls to remote clusters; otherwise returns FALSE.
+         */
+        @JvmStatic
+        fun isRemoteMonitor(monitor: Monitor, localClusterName: String): Boolean {
+            var isRemoteMonitor = false
+            monitor.inputs.forEach inputCheck@{
+                when (it) {
+                    is ClusterMetricsInput -> {
+                        it.clusters.forEach { clusterName ->
+                            if (clusterName != localClusterName) {
+                                isRemoteMonitor = true
+                                return@inputCheck
+                            }
+                        }
+                    }
+                    is SearchInput -> {
+                        // Remote indexes follow the pattern "<CLUSTER_NAME>:<INDEX_NAME>".
+                        // Index entries without a CLUSTER_NAME indicate they're store on the local cluster.
+                        it.indices.forEach { index ->
+                            val clusterName = parseClusterName(index)
+                            if (clusterName != localClusterName) {
+                                isRemoteMonitor = true
+                                return@inputCheck
+                            }
+                        }
+                    }
+                    is DocLevelMonitorInput -> {
+                        // TODO: When document level monitors are supported, this check will be similar to SearchInput.
+                        throw IllegalArgumentException("Per document monitors do not currently support cross-cluster search.")
+                    }
+                    else -> {
+                        throw IllegalArgumentException("Unsupported input type: ${it.name()}.")
+                    }
+                }
+            }
+            return isRemoteMonitor
+        }
+
+        /**
+         * Uses the monitor inputs to determine whether the monitor makes calls to remote clusters.
+         * @param monitor The monitor to evaluate.
+         * @param clusterService Used to retrieve the name of the local cluster.
+         * @return TRUE if the monitor makes calls to remote clusters; otherwise returns FALSE.
+         */
+        @JvmStatic
+        fun isRemoteMonitor(monitor: Monitor, clusterService: ClusterService): Boolean {
+            return isRemoteMonitor(monitor = monitor, localClusterName = clusterService.clusterName.value())
+        }
+
+        /**
+         * Parses the list of indexes into a map of CLUSTER_NAME to List<INDEX_NAME>.
+         * @param indexes A list of index names in "<CLUSTER_NAME>:<INDEX_NAME>" format.
+         * @param localClusterName The name of the local cluster.
+         * @return A map of CLUSTER_NAME to List<INDEX_NAME>
+         */
+        @JvmStatic
+        fun separateClusterIndexes(indexes: List<String>, localClusterName: String): HashMap<String, MutableList<String>> {
+            val output = hashMapOf<String, MutableList<String>>()
+            indexes.forEach { index ->
+                var clusterName = parseClusterName(index)
+                val indexName = parseIndexName(index)
+
+                // If the index entry does not have a CLUSTER_NAME, it indicates the index is on the local cluster.
+                if (clusterName.isEmpty()) clusterName = localClusterName
+
+                output.getOrPut(clusterName) { mutableListOf() }.add(indexName)
+            }
+            return output
+        }
+
+        /**
+         * Parses the list of indexes into a map of CLUSTER_NAME to List<INDEX_NAME>.
+         * @param indexes A list of index names in "<CLUSTER_NAME>:<INDEX_NAME>" format.
+         *      Local indexes can also be in "<INDEX_NAME>" format.
+         * @param clusterService Used to retrieve the name of the local cluster.
+         * @return A map of CLUSTER_NAME to List<INDEX_NAME>
+         */
+        @JvmStatic
+        fun separateClusterIndexes(indexes: List<String>, clusterService: ClusterService): HashMap<String, MutableList<String>> {
+            return separateClusterIndexes(indexes = indexes, localClusterName = clusterService.clusterName.value())
+        }
+
+        /**
+         * The [NodeClient] used by the plugin cannot execute searches against local indexes
+         * using format "<LOCAL_CLUSTER_NAME>:<INDEX_NAME>". That format only supports querying remote indexes.
+         * This function formats a list of indexes to be supplied directly to a [SearchRequest].
+         * @param indexes A list of index names in "<CLUSTER_NAME>:<INDEX_NAME>" format.
+         * @param localClusterName The name of the local cluster.
+         * @return A list of indexes with any remote indexes in "<CLUSTER_NAME>:<INDEX_NAME>" format,
+         *      and any local indexes in "<INDEX_NAME>" format.
+         */
+        @JvmStatic
+        fun parseIndexesForRemoteSearch(indexes: List<String>, localClusterName: String): List<String> {
+            return indexes.map {
+                var index = it
+                val clusterName = parseClusterName(it)
+                if (clusterName.isNotEmpty() && clusterName == localClusterName) {
+                    index = parseIndexName(it)
+                }
+                index
+            }
+        }
+
+        /**
+         * The [NodeClient] used by the plugin cannot execute searches against local indexes
+         * using format "<LOCAL_CLUSTER_NAME>:<INDEX_NAME>". That format only supports querying remote indexes.
+         * This function formats a list of indexes to be supplied directly to a [SearchRequest].
+         * @param indexes A list of index names in "<CLUSTER_NAME>:<INDEX_NAME>" format.
+         * @param clusterService Used to retrieve the name of the local cluster.
+         * @return A list of indexes with any remote indexes in "<CLUSTER_NAME>:<INDEX_NAME>" format,
+         *      and any local indexes in "<INDEX_NAME>" format.
+         */
+        @JvmStatic
+        fun parseIndexesForRemoteSearch(indexes: List<String>, clusterService: ClusterService): List<String> {
+            return parseIndexesForRemoteSearch(indexes = indexes, localClusterName = clusterService.clusterName.value())
+        }
+
+        /**
+         * Uses the clusterName to determine whether the target client is the local or a remote client,
+         * and returns the appropriate client.
+         * @param clusterName The name of the cluster to evaluate.
+         * @param client The local [NodeClient].
+         * @param localClusterName The name of the local cluster.
+         * @return The local [NodeClient] for the local cluster, or a remote client for a remote cluster.
+         */
+        @JvmStatic
+        fun getClientForCluster(clusterName: String, client: Client, localClusterName: String): Client {
+            return if (clusterName == localClusterName) client else client.getRemoteClusterClient(clusterName)
+        }
+
+        /**
+         * Uses the clusterName to determine whether the target client is the local or a remote client,
+         * and returns the appropriate client.
+         * @param clusterName The name of the cluster to evaluate.
+         * @param client The local [NodeClient].
+         * @param clusterService Used to retrieve the name of the local cluster.
+         * @return The local [NodeClient] for the local cluster, or a remote client for a remote cluster.
+         */
+        @JvmStatic
+        fun getClientForCluster(clusterName: String, client: Client, clusterService: ClusterService): Client {
+            return getClientForCluster(clusterName = clusterName, client = client, localClusterName = clusterService.clusterName.value())
+        }
+
+        /**
+         * Uses the index name to determine whether the target client is the local or a remote client,
+         * and returns the appropriate client.
+         * @param index The name of the index to evaluate.
+         *      Can be in either "<CLUSTER_NAME>:<INDEX_NAME>" or "<INDEX_NAME>" format.
+         * @param client The local [NodeClient].
+         * @param localClusterName The name of the local cluster.
+         * @return The local [NodeClient] for the local cluster, or a remote client for a remote cluster.
+         */
+        @JvmStatic
+        fun getClientForIndex(index: String, client: Client, localClusterName: String): Client {
+            val clusterName = parseClusterName(index)
+            return if (clusterName.isNotEmpty() && clusterName != localClusterName)
+                client.getRemoteClusterClient(clusterName) else client
+        }
+
+        /**
+         * Uses the index name to determine whether the target client is the local or a remote client,
+         * and returns the appropriate client.
+         * @param index The name of the index to evaluate.
+         *      Can be in either "<CLUSTER_NAME>:<INDEX_NAME>" or "<INDEX_NAME>" format.
+         * @param client The local [NodeClient].
+         * @param clusterService Used to retrieve the name of the local cluster.
+         * @return The local [NodeClient] for the local cluster, or a remote client for a remote cluster.
+         */
+        @JvmStatic
+        fun getClientForIndex(index: String, client: Client, clusterService: ClusterService): Client {
+            return getClientForIndex(index = index, client = client, localClusterName = clusterService.clusterName.value())
+        }
+
+        /**
+         * @param index The name of the index to evaluate.
+         *      Can be in either "<CLUSTER_NAME>:<INDEX_NAME>" or "<INDEX_NAME>" format.
+         * @return The cluster name if present; else an empty string.
+         */
+        @JvmStatic
+        fun parseClusterName(index: String): String {
+            return if (index.contains(":")) index.split(":").getOrElse(0) { "" }
+            else ""
+        }
+
+        /**
+         * @param index The name of the index to evaluate.
+         *      Can be in either "<CLUSTER_NAME>:<INDEX_NAME>" or "<INDEX_NAME>" format.
+         * @return The index name.
+         */
+        @JvmStatic
+        fun parseIndexName(index: String): String {
+            return if (index.contains(":")) index.split(":").getOrElse(1) { index }
+            else index
+        }
+
+        /**
+         * If clusterName is provided, combines the inputs into "<CLUSTER_NAME>:<INDEX_NAME>" format.
+         * @param clusterName
+         * @param indexName
+         * @return The formatted string.
+         */
+        @JvmStatic
+        fun formatClusterAndIndexName(clusterName: String, indexName: String): String {
+            return if (clusterName.isNotEmpty()) "$clusterName:$indexName"
+            else indexName
+        }
+    }
+}
diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/DocLevelMonitorQueries.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/DocLevelMonitorQueries.kt
index 5b92624a5..52ef1d26f 100644
--- a/alerting/src/main/kotlin/org/opensearch/alerting/util/DocLevelMonitorQueries.kt
+++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/DocLevelMonitorQueries.kt
@@ -13,6 +13,8 @@ import org.opensearch.action.admin.indices.alias.Alias
 import org.opensearch.action.admin.indices.create.CreateIndexRequest
 import org.opensearch.action.admin.indices.create.CreateIndexResponse
 import org.opensearch.action.admin.indices.delete.DeleteIndexRequest
+import org.opensearch.action.admin.indices.exists.indices.IndicesExistsRequest
+import org.opensearch.action.admin.indices.exists.indices.IndicesExistsResponse
 import org.opensearch.action.admin.indices.mapping.put.PutMappingRequest
 import org.opensearch.action.admin.indices.rollover.RolloverRequest
 import org.opensearch.action.admin.indices.rollover.RolloverResponse
@@ -38,8 +40,16 @@ import org.opensearch.commons.alerting.model.DocLevelMonitorInput
 import org.opensearch.commons.alerting.model.DocLevelQuery
 import org.opensearch.commons.alerting.model.Monitor
 import org.opensearch.commons.alerting.model.ScheduledJob
+import org.opensearch.core.action.ActionListener
 import org.opensearch.core.rest.RestStatus
 import org.opensearch.index.mapper.MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING
+import org.opensearch.index.query.QueryBuilders
+import org.opensearch.index.reindex.BulkByScrollResponse
+import org.opensearch.index.reindex.DeleteByQueryAction
+import org.opensearch.index.reindex.DeleteByQueryRequestBuilder
+import kotlin.coroutines.resume
+import kotlin.coroutines.resumeWithException
+import kotlin.coroutines.suspendCoroutine
 
 private val log = LogManager.getLogger(DocLevelMonitorQueries::class.java)
 
@@ -134,6 +144,42 @@ class DocLevelMonitorQueries(private val client: Client, private val clusterServ
         return true
     }
 
+    suspend fun deleteDocLevelQueriesOnDryRun(monitorMetadata: MonitorMetadata) {
+        try {
+            monitorMetadata.sourceToQueryIndexMapping.forEach { (_, queryIndex) ->
+                val indicesExistsResponse: IndicesExistsResponse =
+                    client.suspendUntil {
+                        client.admin().indices().exists(IndicesExistsRequest(queryIndex), it)
+                    }
+                if (indicesExistsResponse.isExists == false) {
+                    return
+                }
+
+                val queryBuilder = QueryBuilders.boolQuery()
+                    .must(QueryBuilders.existsQuery("monitor_id"))
+                    .mustNot(QueryBuilders.wildcardQuery("monitor_id", "*"))
+
+                val response: BulkByScrollResponse = suspendCoroutine { cont ->
+                    DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE)
+                        .source(queryIndex)
+                        .filter(queryBuilder)
+                        .refresh(true)
+                        .execute(
+                            object : ActionListener<BulkByScrollResponse> {
+                                override fun onResponse(response: BulkByScrollResponse) = cont.resume(response)
+                                override fun onFailure(t: Exception) = cont.resumeWithException(t)
+                            }
+                        )
+                }
+                response.bulkFailures.forEach {
+                    log.error("Failed deleting queries while removing dry run queries: [${it.id}] cause: [${it.cause}] ")
+                }
+            }
+        } catch (e: Exception) {
+            log.error("Failed to delete doc level queries on dry run", e)
+        }
+    }
+
     fun docLevelQueryIndexExists(dataSources: DataSources): Boolean {
         val clusterState = clusterService.state()
         return clusterState.metadata.hasAlias(dataSources.queryIndex)
@@ -207,11 +253,25 @@ class DocLevelMonitorQueries(private val client: Client, private val clusterServ
 
         // Run through each backing index and apply appropriate mappings to query index
         indices.forEach { indexName ->
-            val concreteIndices = IndexUtils.resolveAllIndices(
+            var concreteIndices = IndexUtils.resolveAllIndices(
                 listOf(indexName),
                 monitorCtx.clusterService!!,
                 monitorCtx.indexNameExpressionResolver!!
             )
+            if (IndexUtils.isAlias(indexName, monitorCtx.clusterService!!.state()) ||
+                IndexUtils.isDataStream(indexName, monitorCtx.clusterService!!.state())
+            ) {
+                val lastWriteIndex = concreteIndices.find { monitorMetadata.lastRunContext.containsKey(it) }
+                if (lastWriteIndex != null) {
+                    val lastWriteIndexCreationDate =
+                        IndexUtils.getCreationDateForIndex(lastWriteIndex, monitorCtx.clusterService!!.state())
+                    concreteIndices = IndexUtils.getNewestIndicesByCreationDate(
+                        concreteIndices,
+                        monitorCtx.clusterService!!.state(),
+                        lastWriteIndexCreationDate
+                    )
+                }
+            }
             val updatedIndexName = indexName.replace("*", "_")
             val updatedProperties = mutableMapOf<String, Any>()
             val allFlattenPaths = mutableSetOf<Pair<String, String>>()
@@ -336,6 +396,7 @@ class DocLevelMonitorQueries(private val client: Client, private val clusterServ
             var query = it.query
             conflictingPaths.forEach { conflictingPath ->
                 if (query.contains(conflictingPath)) {
+                    query = transformExistsQuery(query, conflictingPath, "<index>", monitorId)
                     query = query.replace("$conflictingPath:", "${conflictingPath}_<index>_$monitorId:")
                     filteredConcreteIndices.addAll(conflictingPathToConcreteIndices[conflictingPath]!!)
                 }
@@ -358,6 +419,7 @@ class DocLevelMonitorQueries(private val client: Client, private val clusterServ
             var query = it.query
             flattenPaths.forEach { fieldPath ->
                 if (!conflictingPaths.contains(fieldPath.first)) {
+                    query = transformExistsQuery(query, fieldPath.first, sourceIndex, monitorId)
                     query = query.replace("${fieldPath.first}:", "${fieldPath.first}_${sourceIndex}_$monitorId:")
                 }
             }
@@ -388,6 +450,26 @@ class DocLevelMonitorQueries(private val client: Client, private val clusterServ
         }
     }
 
+    /**
+     * Transforms the query if it includes an _exists_ clause to append the index name and the monitor id to the field value
+     */
+    private fun transformExistsQuery(query: String, conflictingPath: String, indexName: String, monitorId: String): String {
+        return query
+            .replace("_exists_: ", "_exists_:") // remove space to read exists query as one string
+            .split("\\s+".toRegex())
+            .joinToString(separator = " ") { segment ->
+                if (segment.contains("_exists_:")) {
+                    val trimSegement = segment.trim { it == '(' || it == ')' } // remove any delimiters from ends
+                    val (_, value) = trimSegement.split(":", limit = 2) // split into key and value
+                    val newString = if (value == conflictingPath)
+                        segment.replace(conflictingPath, "${conflictingPath}_${indexName}_$monitorId") else segment
+                    newString
+                } else {
+                    segment
+                }
+            }
+    }
+
     private suspend fun updateQueryIndexMappings(
         monitor: Monitor,
         monitorMetadata: MonitorMetadata,
diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/IndexUtils.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/IndexUtils.kt
index 4e34ed3c2..387f5cb22 100644
--- a/alerting/src/main/kotlin/org/opensearch/alerting/util/IndexUtils.kt
+++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/IndexUtils.kt
@@ -12,6 +12,7 @@ import org.opensearch.alerting.alerts.AlertIndices
 import org.opensearch.alerting.core.ScheduledJobIndices
 import org.opensearch.client.IndicesAdminClient
 import org.opensearch.cluster.ClusterState
+import org.opensearch.cluster.metadata.IndexAbstraction
 import org.opensearch.cluster.metadata.IndexMetadata
 import org.opensearch.cluster.metadata.IndexNameExpressionResolver
 import org.opensearch.cluster.service.ClusterService
@@ -153,5 +154,47 @@ class IndexUtils {
 
             return result
         }
+
+        @JvmStatic
+        fun isDataStream(name: String, clusterState: ClusterState): Boolean {
+            return clusterState.metadata().dataStreams().containsKey(name)
+        }
+
+        @JvmStatic
+        fun isAlias(name: String, clusterState: ClusterState): Boolean {
+            return clusterState.metadata().hasAlias(name)
+        }
+
+        @JvmStatic
+        fun getWriteIndex(index: String, clusterState: ClusterState): String? {
+            if (isAlias(index, clusterState) || isDataStream(index, clusterState)) {
+                val metadata = clusterState.metadata.indicesLookup[index]?.writeIndex
+                if (metadata != null) {
+                    return metadata.index.name
+                }
+            }
+            return null
+        }
+
+        @JvmStatic
+        fun getNewestIndicesByCreationDate(concreteIndices: List<String>, clusterState: ClusterState, thresholdDate: Long): List<String> {
+            val filteredIndices = mutableListOf<String>()
+            val lookup = clusterState.metadata().indicesLookup
+            concreteIndices.forEach { indexName ->
+                val index = lookup[indexName]
+                val indexMetadata = clusterState.metadata.index(indexName)
+                if (index != null && index.type == IndexAbstraction.Type.CONCRETE_INDEX) {
+                    if (indexMetadata.creationDate >= thresholdDate) {
+                        filteredIndices.add(indexName)
+                    }
+                }
+            }
+            return filteredIndices
+        }
+
+        @JvmStatic
+        fun getCreationDateForIndex(index: String, clusterState: ClusterState): Long {
+            return clusterState.metadata.index(index).creationDate
+        }
     }
 }
diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/workflow/CompositeWorkflowRunner.kt b/alerting/src/main/kotlin/org/opensearch/alerting/workflow/CompositeWorkflowRunner.kt
index 94e8b9bc3..e68896bc2 100644
--- a/alerting/src/main/kotlin/org/opensearch/alerting/workflow/CompositeWorkflowRunner.kt
+++ b/alerting/src/main/kotlin/org/opensearch/alerting/workflow/CompositeWorkflowRunner.kt
@@ -255,7 +255,7 @@ object CompositeWorkflowRunner : WorkflowRunner() {
                 executionId
             )
         } else if (delegateMonitor.isDocLevelMonitor()) {
-            return DocumentLevelMonitorRunner.runMonitor(
+            return DocumentLevelMonitorRunner().runMonitor(
                 delegateMonitor,
                 monitorCtx,
                 periodStart,
diff --git a/alerting/src/main/resources/org/opensearch/alerting/alerts/alert_mapping.json b/alerting/src/main/resources/org/opensearch/alerting/alerts/alert_mapping.json
index 53fb5b0a2..76e5104cc 100644
--- a/alerting/src/main/resources/org/opensearch/alerting/alerts/alert_mapping.json
+++ b/alerting/src/main/resources/org/opensearch/alerting/alerts/alert_mapping.json
@@ -169,6 +169,14 @@
           "type": "text"
         }
       }
+    },
+    "clusters": {
+      "type" : "text",
+      "fields" : {
+        "keyword" : {
+          "type" : "keyword"
+        }
+      }
     }
   }
 }
\ No newline at end of file
diff --git a/alerting/src/main/resources/org/opensearch/alerting/alerts/finding_mapping.json b/alerting/src/main/resources/org/opensearch/alerting/alerts/finding_mapping.json
index d2ecc0907..1bfea4ebc 100644
--- a/alerting/src/main/resources/org/opensearch/alerting/alerts/finding_mapping.json
+++ b/alerting/src/main/resources/org/opensearch/alerting/alerts/finding_mapping.json
@@ -49,6 +49,9 @@
         },
         "fields": {
           "type": "text"
+        },
+        "query_field_names": {
+          "type": "keyword"
         }
       }
     },
diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/AlertingRestTestCase.kt b/alerting/src/test/kotlin/org/opensearch/alerting/AlertingRestTestCase.kt
index 0e4216c69..50cae9d8c 100644
--- a/alerting/src/test/kotlin/org/opensearch/alerting/AlertingRestTestCase.kt
+++ b/alerting/src/test/kotlin/org/opensearch/alerting/AlertingRestTestCase.kt
@@ -76,6 +76,8 @@ import javax.management.MBeanServerInvocationHandler
 import javax.management.ObjectName
 import javax.management.remote.JMXConnectorFactory
 import javax.management.remote.JMXServiceURL
+import kotlin.collections.ArrayList
+import kotlin.collections.HashMap
 
 /**
  * Superclass for tests that interact with an external test cluster using OpenSearch's RestClient
@@ -911,7 +913,7 @@ abstract class AlertingRestTestCase : ODFERestTestCase() {
     private fun indexDoc(client: RestClient, index: String, id: String, doc: String, refresh: Boolean = true): Response {
         val requestBody = StringEntity(doc, APPLICATION_JSON)
         val params = if (refresh) mapOf("refresh" to "true") else mapOf()
-        val response = client.makeRequest("PUT", "$index/_doc/$id", params, requestBody)
+        val response = client.makeRequest("POST", "$index/_doc/$id?op_type=create", params, requestBody)
         assertTrue(
             "Unable to index doc: '${doc.take(15)}...' to index: '$index'",
             listOf(RestStatus.OK, RestStatus.CREATED).contains(response.restStatus())
@@ -947,6 +949,11 @@ abstract class AlertingRestTestCase : ODFERestTestCase() {
         return index
     }
 
+    protected fun createTestIndex(index: String, mapping: String?, alias: String): String {
+        createIndex(index, Settings.EMPTY, mapping?.trimIndent(), alias)
+        return index
+    }
+
     protected fun createTestConfigIndex(index: String = "." + randomAlphaOfLength(10).lowercase(Locale.ROOT)): String {
         try {
             createIndex(
@@ -983,7 +990,7 @@ abstract class AlertingRestTestCase : ODFERestTestCase() {
         val indicesMap = mutableMapOf<String, Boolean>()
         val indicesJson = jsonBuilder().startObject().startArray("actions")
         indices.keys.map {
-            val indexName = createTestIndex(index = it.lowercase(Locale.ROOT), mapping = "")
+            val indexName = createTestIndex(index = it, mapping = "")
             val isWriteIndex = indices.getOrDefault(indexName, false)
             indicesMap[indexName] = isWriteIndex
             val indexMap = mapOf(
@@ -1000,17 +1007,155 @@ abstract class AlertingRestTestCase : ODFERestTestCase() {
         return mutableMapOf(alias to indicesMap)
     }
 
+    protected fun createDataStream(datastream: String, mappings: String?, useComponentTemplate: Boolean) {
+        val indexPattern = "$datastream*"
+        var componentTemplateMappings = "\"properties\": {" +
+            "  \"netflow.destination_transport_port\":{ \"type\": \"long\" }," +
+            "  \"netflow.destination_ipv4_address\":{ \"type\": \"ip\" }" +
+            "}"
+        if (mappings != null) {
+            componentTemplateMappings = mappings
+        }
+        if (useComponentTemplate) {
+            // Setup index_template
+            createComponentTemplateWithMappings(
+                "my_ds_component_template-$datastream",
+                componentTemplateMappings
+            )
+        }
+        createComposableIndexTemplate(
+            "my_index_template_ds-$datastream",
+            listOf(indexPattern),
+            (if (useComponentTemplate) "my_ds_component_template-$datastream" else null),
+            mappings,
+            true,
+            0
+        )
+        createDataStream(datastream)
+    }
+
+    protected fun createDataStream(datastream: String? = randomAlphaOfLength(10).lowercase(Locale.ROOT)) {
+        client().makeRequest("PUT", "_data_stream/$datastream")
+    }
+
+    protected fun deleteDataStream(datastream: String) {
+        client().makeRequest("DELETE", "_data_stream/$datastream")
+    }
+
+    protected fun createIndexAlias(alias: String, mappings: String?) {
+        val indexPattern = "$alias*"
+        var componentTemplateMappings = "\"properties\": {" +
+            "  \"netflow.destination_transport_port\":{ \"type\": \"long\" }," +
+            "  \"netflow.destination_ipv4_address\":{ \"type\": \"ip\" }" +
+            "}"
+        if (mappings != null) {
+            componentTemplateMappings = mappings
+        }
+        createComponentTemplateWithMappings(
+            "my_alias_component_template-$alias",
+            componentTemplateMappings
+        )
+        createComposableIndexTemplate(
+            "my_index_template_alias-$alias",
+            listOf(indexPattern),
+            "my_alias_component_template-$alias",
+            mappings,
+            false,
+            0
+        )
+        createTestIndex(
+            "$alias-000001",
+            null,
+            """
+            "$alias": {
+              "is_write_index": true
+            }
+            """.trimIndent()
+        )
+    }
+
+    protected fun deleteIndexAlias(alias: String) {
+        client().makeRequest("DELETE", "$alias*/_alias/$alias")
+    }
+
+    protected fun createComponentTemplateWithMappings(componentTemplateName: String, mappings: String?) {
+        val body = """{"template" : {        "mappings": {$mappings}    }}"""
+        client().makeRequest(
+            "PUT",
+            "_component_template/$componentTemplateName",
+            emptyMap(),
+            StringEntity(body, ContentType.APPLICATION_JSON),
+            BasicHeader("Content-Type", "application/json")
+        )
+    }
+
+    protected fun createComposableIndexTemplate(
+        templateName: String,
+        indexPatterns: List<String>,
+        componentTemplateName: String?,
+        mappings: String?,
+        isDataStream: Boolean,
+        priority: Int
+    ) {
+        var body = "{\n"
+        if (isDataStream) {
+            body += "\"data_stream\": { },"
+        }
+        body += "\"index_patterns\": [" +
+            indexPatterns.stream().collect(
+                Collectors.joining(",", "\"", "\"")
+            ) + "],"
+        if (componentTemplateName == null) {
+            body += "\"template\": {\"mappings\": {$mappings}},"
+        }
+        if (componentTemplateName != null) {
+            body += "\"composed_of\": [\"$componentTemplateName\"],"
+        }
+        body += "\"priority\":$priority}"
+        client().makeRequest(
+            "PUT",
+            "_index_template/$templateName",
+            emptyMap(),
+            StringEntity(body, APPLICATION_JSON),
+            BasicHeader("Content-Type", "application/json")
+        )
+    }
+
+    protected fun getDatastreamWriteIndex(datastream: String): String {
+        val response = client().makeRequest("GET", "_data_stream/$datastream", emptyMap(), null)
+        var respAsMap = responseAsMap(response)
+        if (respAsMap.containsKey("data_streams")) {
+            respAsMap = (respAsMap["data_streams"] as ArrayList<HashMap<String, *>>)[0]
+            val indices = respAsMap["indices"] as List<Map<String, Any>>
+            val index = indices.last()
+            return index["index_name"] as String
+        } else {
+            respAsMap = respAsMap[datastream] as Map<String, Object>
+        }
+        val indices = respAsMap["indices"] as Array<String>
+        return indices.last()
+    }
+
+    protected fun rolloverDatastream(datastream: String) {
+        client().makeRequest(
+            "POST",
+            datastream + "/_rollover",
+            emptyMap(),
+            null
+        )
+    }
+
     protected fun randomAliasIndices(
         alias: String,
         num: Int = randomIntBetween(1, 10),
         includeWriteIndex: Boolean = true,
     ): Map<String, Boolean> {
         val indices = mutableMapOf<String, Boolean>()
-        val writeIndex = randomIntBetween(0, num)
+        val writeIndex = randomIntBetween(0, num - 1)
         for (i: Int in 0 until num) {
-            var indexName = randomAlphaOfLength(10)
+            var indexName = randomAlphaOfLength(10).lowercase(Locale.ROOT)
             while (indexName.equals(alias) || indices.containsKey(indexName))
-                indexName = randomAlphaOfLength(10)
+                indexName = randomAlphaOfLength(10).lowercase(Locale.ROOT)
             indices[indexName] = includeWriteIndex && i == writeIndex
         }
         return indices
diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/DocumentMonitorRunnerIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/DocumentMonitorRunnerIT.kt
index 9f9164065..0f6218f30 100644
--- a/alerting/src/test/kotlin/org/opensearch/alerting/DocumentMonitorRunnerIT.kt
+++ b/alerting/src/test/kotlin/org/opensearch/alerting/DocumentMonitorRunnerIT.kt
@@ -10,6 +10,8 @@ import org.apache.http.entity.StringEntity
 import org.opensearch.action.search.SearchResponse
 import org.opensearch.alerting.alerts.AlertIndices.Companion.ALL_ALERT_INDEX_PATTERN
 import org.opensearch.alerting.alerts.AlertIndices.Companion.ALL_FINDING_INDEX_PATTERN
+import org.opensearch.alerting.core.lock.LockService
+import org.opensearch.alerting.settings.AlertingSettings
 import org.opensearch.client.Response
 import org.opensearch.client.ResponseException
 import org.opensearch.common.xcontent.json.JsonXContent
@@ -17,16 +19,20 @@ import org.opensearch.commons.alerting.model.Alert
 import org.opensearch.commons.alerting.model.DataSources
 import org.opensearch.commons.alerting.model.DocLevelMonitorInput
 import org.opensearch.commons.alerting.model.DocLevelQuery
+import org.opensearch.commons.alerting.model.IntervalSchedule
 import org.opensearch.commons.alerting.model.action.ActionExecutionPolicy
 import org.opensearch.commons.alerting.model.action.AlertCategory
 import org.opensearch.commons.alerting.model.action.PerAlertActionScope
 import org.opensearch.commons.alerting.model.action.PerExecutionActionScope
 import org.opensearch.core.rest.RestStatus
 import org.opensearch.script.Script
+import org.opensearch.test.OpenSearchTestCase
 import java.time.ZonedDateTime
 import java.time.format.DateTimeFormatter
+import java.time.temporal.ChronoUnit
 import java.time.temporal.ChronoUnit.MILLIS
 import java.util.Locale
+import java.util.concurrent.TimeUnit
 
 class DocumentMonitorRunnerIT : AlertingRestTestCase() {
 
@@ -73,10 +79,25 @@ class DocumentMonitorRunnerIT : AlertingRestTestCase() {
 
         val alerts = searchAlerts(monitor)
         assertEquals("Alert saved for test monitor", 0, alerts.size)
+
+        // ensure doc level query is deleted on dry run
+        val request = """{
+            "size": 10,
+            "query": {
+                "match_all": {}
+            }
+        }"""
+        var httpResponse = adminClient().makeRequest(
+            "GET", "/${monitor.dataSources.queryIndex}/_search",
+            StringEntity(request, ContentType.APPLICATION_JSON)
+        )
+        assertEquals("Search failed", RestStatus.OK, httpResponse.restStatus())
+        var searchResponse = SearchResponse.fromXContent(createParser(JsonXContent.jsonXContent, httpResponse.entity.content))
+        searchResponse.hits.totalHits?.let { assertEquals("Query saved in query index", 0L, it.value) }
     }
 
-    fun `test execute monitor returns search result with dryrun`() {
-        val testIndex = createTestIndex()
+    fun `test dryrun execute monitor with queryFieldNames set up with correct field`() {
+
         val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(MILLIS))
         val testDoc = """{
             "message" : "This is an error from IAD region",
@@ -84,31 +105,45 @@ class DocumentMonitorRunnerIT : AlertingRestTestCase() {
             "test_field" : "us-west-2"
         }"""
 
-        val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", name = "3", fields = listOf())
-        val docLevelInput = DocLevelMonitorInput("description", listOf(testIndex), listOf(docQuery))
+        val index = createTestIndex()
 
-        val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN)
-        val monitor = randomDocumentLevelMonitor(inputs = listOf(docLevelInput), triggers = listOf(trigger))
+        val docQuery =
+            DocLevelQuery(query = "test_field:\"us-west-2\"", name = "3", fields = listOf(), queryFieldNames = listOf("test_field"))
+        val docLevelInput = DocLevelMonitorInput("description", listOf(index), listOf(docQuery))
 
-        indexDoc(testIndex, "1", testDoc)
-        indexDoc(testIndex, "5", testDoc)
+        val action = randomAction(template = randomTemplateScript("Hello {{ctx.monitor.name}}"), destinationId = createDestination().id)
+        val monitor = randomDocumentLevelMonitor(
+            inputs = listOf(docLevelInput),
+            triggers = listOf(randomDocumentLevelTrigger(condition = ALWAYS_RUN, actions = listOf(action)))
+        )
+
+        indexDoc(index, "1", testDoc)
 
         val response = executeMonitor(monitor, params = DRYRUN_MONITOR)
 
         val output = entityAsMap(response)
-
         assertEquals(monitor.name, output["monitor_name"])
-        @Suppress("UNCHECKED_CAST")
-        val searchResult = (output.objectMap("input_results")["results"] as List<Map<String, Any>>).first()
-        @Suppress("UNCHECKED_CAST")
-        val matchingDocsToQuery = searchResult[docQuery.id] as List<String>
-        assertEquals("Incorrect search result", 2, matchingDocsToQuery.size)
-        assertTrue("Incorrect search result", matchingDocsToQuery.contains("1|$testIndex"))
-        assertTrue("Incorrect search result", matchingDocsToQuery.contains("5|$testIndex"))
+
+        assertEquals(1, output.objectMap("trigger_results").values.size)
+
+        for (triggerResult in output.objectMap("trigger_results").values) {
+            assertEquals(1, triggerResult.objectMap("action_results").values.size)
+            for (alertActionResult in triggerResult.objectMap("action_results").values) {
+                for (actionResult in alertActionResult.values) {
+                    @Suppress("UNCHECKED_CAST") val actionOutput = (actionResult as Map<String, Map<String, String>>)["output"]
+                        as Map<String, String>
+                    assertEquals("Hello ${monitor.name}", actionOutput["subject"])
+                    assertEquals("Hello ${monitor.name}", actionOutput["message"])
+                }
+            }
+        }
+
+        val alerts = searchAlerts(monitor)
+        assertEquals("Alert saved for test monitor", 0, alerts.size)
     }
 
-    fun `test execute monitor generates alerts and findings`() {
-        val testIndex = createTestIndex()
+    fun `test seq_no calculation correctness when docs are deleted`() {
+        adminClient().updateSettings(AlertingSettings.DOC_LEVEL_MONITOR_SHARD_FETCH_SIZE.key, 2)
         val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(MILLIS))
         val testDoc = """{
             "message" : "This is an error from IAD region",
@@ -116,39 +151,42 @@ class DocumentMonitorRunnerIT : AlertingRestTestCase() {
             "test_field" : "us-west-2"
         }"""
 
-        val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", name = "3", fields = listOf())
-        val docLevelInput = DocLevelMonitorInput("description", listOf(testIndex), listOf(docQuery))
+        val index = createTestIndex()
 
-        val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN)
-        val monitor = createMonitor(randomDocumentLevelMonitor(inputs = listOf(docLevelInput), triggers = listOf(trigger)))
-        assertNotNull(monitor.id)
+        val docQuery =
+            DocLevelQuery(query = "test_field:\"us-west-2\"", name = "3", fields = listOf())
+        val docLevelInput = DocLevelMonitorInput("description", listOf(index), listOf(docQuery))
 
-        indexDoc(testIndex, "1", testDoc)
-        indexDoc(testIndex, "5", testDoc)
+        val action = randomAction(template = randomTemplateScript("Hello {{ctx.monitor.name}}"), destinationId = createDestination().id)
+        val monitor = randomDocumentLevelMonitor(
+            inputs = listOf(docLevelInput),
+            triggers = listOf(randomDocumentLevelTrigger(condition = ALWAYS_RUN, actions = listOf(action)))
+        )
 
-        val response = executeMonitor(monitor.id)
+        indexDoc(index, "1", testDoc)
+        indexDoc(index, "2", testDoc)
+        indexDoc(index, "3", testDoc)
+        indexDoc(index, "4", testDoc)
+        indexDoc(index, "5", testDoc)
+        indexDoc(index, "11", testDoc)
+        indexDoc(index, "21", testDoc)
+        indexDoc(index, "31", testDoc)
+        indexDoc(index, "41", testDoc)
+        indexDoc(index, "51", testDoc)
+
+        deleteDoc(index, "51")
+        val response = executeMonitor(monitor, params = mapOf("dryrun" to "false"))
 
         val output = entityAsMap(response)
-
         assertEquals(monitor.name, output["monitor_name"])
-        @Suppress("UNCHECKED_CAST")
-        val searchResult = (output.objectMap("input_results")["results"] as List<Map<String, Any>>).first()
-        @Suppress("UNCHECKED_CAST")
-        val matchingDocsToQuery = searchResult[docQuery.id] as List<String>
-        assertEquals("Incorrect search result", 2, matchingDocsToQuery.size)
-        assertTrue("Incorrect search result", matchingDocsToQuery.containsAll(listOf("1|$testIndex", "5|$testIndex")))
-
-        val alerts = searchAlertsWithFilter(monitor)
-        assertEquals("Alert saved for test monitor", 2, alerts.size)
 
-        val findings = searchFindings(monitor)
-        assertEquals("Findings saved for test monitor", 2, findings.size)
-        assertTrue("Findings saved for test monitor", findings[0].relatedDocIds.contains("1"))
-        assertTrue("Findings saved for test monitor", findings[1].relatedDocIds.contains("5"))
+        for (triggerResult in output.objectMap("trigger_results").values) {
+            assertEquals(9, triggerResult.objectMap("action_results").values.size)
+        }
     }
 
-    fun `test execute monitor with tag as trigger condition generates alerts and findings`() {
-        val testIndex = createTestIndex()
+    fun `test dryrun execute monitor with queryFieldNames set up with wrong field`() {
+
         val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(MILLIS))
         val testDoc = """{
             "message" : "This is an error from IAD region",
@@ -156,39 +194,53 @@ class DocumentMonitorRunnerIT : AlertingRestTestCase() {
             "test_field" : "us-west-2"
         }"""
 
-        val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", name = "3", tags = listOf("test_tag"), fields = listOf())
-        val docLevelInput = DocLevelMonitorInput("description", listOf(testIndex), listOf(docQuery))
+        val index = createTestIndex()
+        // using wrong field name
+        val docQuery = DocLevelQuery(
+            query = "test_field:\"us-west-2\"",
+            name = "3",
+            fields = listOf(),
+            queryFieldNames = listOf("wrong_field")
+        )
+        val docLevelInput = DocLevelMonitorInput("description", listOf(index), listOf(docQuery))
 
-        val trigger = randomDocumentLevelTrigger(condition = Script("query[tag=test_tag]"))
-        val monitor = createMonitor(randomDocumentLevelMonitor(inputs = listOf(docLevelInput), triggers = listOf(trigger)))
-        assertNotNull(monitor.id)
+        val action = randomAction(template = randomTemplateScript("Hello {{ctx.monitor.name}}"), destinationId = createDestination().id)
+        val monitor = randomDocumentLevelMonitor(
+            inputs = listOf(docLevelInput),
+            triggers = listOf(randomDocumentLevelTrigger(condition = ALWAYS_RUN, actions = listOf(action)))
+        )
 
-        indexDoc(testIndex, "1", testDoc)
-        indexDoc(testIndex, "5", testDoc)
+        indexDoc(index, "1", testDoc)
+        indexDoc(index, "2", testDoc)
+        indexDoc(index, "3", testDoc)
+        indexDoc(index, "4", testDoc)
+        indexDoc(index, "5", testDoc)
 
-        val response = executeMonitor(monitor.id)
+        val response = executeMonitor(monitor, params = DRYRUN_MONITOR)
 
         val output = entityAsMap(response)
-
         assertEquals(monitor.name, output["monitor_name"])
-        @Suppress("UNCHECKED_CAST")
-        val searchResult = (output.objectMap("input_results")["results"] as List<Map<String, Any>>).first()
-        @Suppress("UNCHECKED_CAST")
-        val matchingDocsToQuery = searchResult[docQuery.id] as List<String>
-        assertEquals("Incorrect search result", 2, matchingDocsToQuery.size)
-        assertTrue("Incorrect search result", matchingDocsToQuery.containsAll(listOf("1|$testIndex", "5|$testIndex")))
 
-        val alerts = searchAlertsWithFilter(monitor)
-        assertEquals("Alert saved for test monitor", 2, alerts.size)
+        assertEquals(1, output.objectMap("trigger_results").values.size)
 
-        val findings = searchFindings(monitor)
-        assertEquals("Findings saved for test monitor", 2, findings.size)
-        assertTrue("Findings saved for test monitor", findings[0].relatedDocIds.contains("1"))
-        assertTrue("Findings saved for test monitor", findings[1].relatedDocIds.contains("5"))
+        for (triggerResult in output.objectMap("trigger_results").values) {
+            assertEquals(0, triggerResult.objectMap("action_results").values.size)
+            for (alertActionResult in triggerResult.objectMap("action_results").values) {
+                for (actionResult in alertActionResult.values) {
+                    @Suppress("UNCHECKED_CAST") val actionOutput = (actionResult as Map<String, Map<String, String>>)["output"]
+                        as Map<String, String>
+                    assertEquals("Hello ${monitor.name}", actionOutput["subject"])
+                    assertEquals("Hello ${monitor.name}", actionOutput["message"])
+                }
+            }
+        }
+
+        val alerts = searchAlerts(monitor)
+        assertEquals("Alert saved for test monitor", 0, alerts.size)
     }
 
-    fun `test execute monitor input error`() {
-        val testIndex = createTestIndex()
+    fun `test fetch_query_field_names setting is disabled by configuring queryFieldNames set up with wrong field still works`() {
+        adminClient().updateSettings(AlertingSettings.DOC_LEVEL_MONITOR_FETCH_ONLY_QUERY_FIELDS_ENABLED.key, "false")
         val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(MILLIS))
         val testDoc = """{
             "message" : "This is an error from IAD region",
@@ -196,29 +248,48 @@ class DocumentMonitorRunnerIT : AlertingRestTestCase() {
             "test_field" : "us-west-2"
         }"""
 
-        val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", name = "3", tags = listOf("test_tag"), fields = listOf())
-        val docLevelInput = DocLevelMonitorInput("description", listOf(testIndex), listOf(docQuery))
+        val index = createTestIndex()
+        // using wrong field name
+        val docQuery = DocLevelQuery(
+            query = "test_field:\"us-west-2\"",
+            name = "3",
+            fields = listOf(),
+            queryFieldNames = listOf("wrong_field")
+        )
+        val docLevelInput = DocLevelMonitorInput("description", listOf(index), listOf(docQuery))
 
-        val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN)
-        val monitor = createMonitor(randomDocumentLevelMonitor(inputs = listOf(docLevelInput), triggers = listOf(trigger)))
-        assertNotNull(monitor.id)
+        val action = randomAction(template = randomTemplateScript("Hello {{ctx.monitor.name}}"), destinationId = createDestination().id)
+        val monitor = randomDocumentLevelMonitor(
+            inputs = listOf(docLevelInput),
+            triggers = listOf(randomDocumentLevelTrigger(condition = ALWAYS_RUN, actions = listOf(action)))
+        )
 
-        deleteIndex(testIndex)
+        indexDoc(index, "1", testDoc)
 
-        val response = executeMonitor(monitor.id)
+        val response = executeMonitor(monitor, params = DRYRUN_MONITOR)
 
         val output = entityAsMap(response)
         assertEquals(monitor.name, output["monitor_name"])
-        @Suppress("UNCHECKED_CAST")
-        val inputResults = output.stringMap("input_results")
-        assertTrue("Missing monitor error message", (inputResults?.get("error") as String).isNotEmpty())
+
+        assertEquals(1, output.objectMap("trigger_results").values.size)
+
+        for (triggerResult in output.objectMap("trigger_results").values) {
+            assertEquals(1, triggerResult.objectMap("action_results").values.size)
+            for (alertActionResult in triggerResult.objectMap("action_results").values) {
+                for (actionResult in alertActionResult.values) {
+                    @Suppress("UNCHECKED_CAST") val actionOutput = (actionResult as Map<String, Map<String, String>>)["output"]
+                        as Map<String, String>
+                    assertEquals("Hello ${monitor.name}", actionOutput["subject"])
+                    assertEquals("Hello ${monitor.name}", actionOutput["message"])
+                }
+            }
+        }
 
         val alerts = searchAlerts(monitor)
-        assertEquals("Alert not saved", 1, alerts.size)
-        assertEquals("Alert status is incorrect", Alert.State.ERROR, alerts[0].state)
+        assertEquals("Alert saved for test monitor", 0, alerts.size)
     }
 
-    fun `test execute monitor generates alerts and findings with per alert execution for actions`() {
+    fun `test execute monitor returns search result with dryrun`() {
         val testIndex = createTestIndex()
         val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(MILLIS))
         val testDoc = """{
@@ -230,27 +301,13 @@ class DocumentMonitorRunnerIT : AlertingRestTestCase() {
         val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", name = "3", fields = listOf())
         val docLevelInput = DocLevelMonitorInput("description", listOf(testIndex), listOf(docQuery))
 
-        val alertCategories = AlertCategory.values()
-        val actionExecutionScope = PerAlertActionScope(
-            actionableAlerts = (1..randomInt(alertCategories.size)).map { alertCategories[it - 1] }.toSet()
-        )
-        val actionExecutionPolicy = ActionExecutionPolicy(actionExecutionScope)
-        val actions = (0..randomInt(10)).map {
-            randomActionWithPolicy(
-                template = randomTemplateScript("Hello {{ctx.monitor.name}}"),
-                destinationId = createDestination().id,
-                actionExecutionPolicy = actionExecutionPolicy
-            )
-        }
-
-        val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN, actions = actions)
-        val monitor = createMonitor(randomDocumentLevelMonitor(inputs = listOf(docLevelInput), triggers = listOf(trigger)))
-        assertNotNull(monitor.id)
+        val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN)
+        val monitor = randomDocumentLevelMonitor(inputs = listOf(docLevelInput), triggers = listOf(trigger))
 
         indexDoc(testIndex, "1", testDoc)
         indexDoc(testIndex, "5", testDoc)
 
-        val response = executeMonitor(monitor.id)
+        val response = executeMonitor(monitor, params = DRYRUN_MONITOR)
 
         val output = entityAsMap(response)
 
@@ -260,33 +317,26 @@ class DocumentMonitorRunnerIT : AlertingRestTestCase() {
         @Suppress("UNCHECKED_CAST")
         val matchingDocsToQuery = searchResult[docQuery.id] as List<String>
         assertEquals("Incorrect search result", 2, matchingDocsToQuery.size)
-        assertTrue("Incorrect search result", matchingDocsToQuery.containsAll(listOf("1|$testIndex", "5|$testIndex")))
+        assertTrue("Incorrect search result", matchingDocsToQuery.contains("1|$testIndex"))
+        assertTrue("Incorrect search result", matchingDocsToQuery.contains("5|$testIndex"))
 
-        for (triggerResult in output.objectMap("trigger_results").values) {
-            assertEquals(2, triggerResult.objectMap("action_results").values.size)
-            for (alertActionResult in triggerResult.objectMap("action_results").values) {
-                assertEquals(actions.size, alertActionResult.values.size)
-                for (actionResult in alertActionResult.values) {
-                    @Suppress("UNCHECKED_CAST") val actionOutput = (actionResult as Map<String, Map<String, String>>)["output"]
-                        as Map<String, String>
-                    assertEquals("Hello ${monitor.name}", actionOutput["subject"])
-                    assertEquals("Hello ${monitor.name}", actionOutput["message"])
-                }
+        // ensure doc level query is deleted on dry run
+        val request = """{
+            "size": 10,
+            "query": {
+                "match_all": {}
             }
-        }
-
-        refreshAllIndices()
-
-        val alerts = searchAlertsWithFilter(monitor)
-        assertEquals("Alert saved for test monitor", 2, alerts.size)
-
-        val findings = searchFindings(monitor)
-        assertEquals("Findings saved for test monitor", 2, findings.size)
-        assertTrue("Findings saved for test monitor", findings[0].relatedDocIds.contains("1"))
-        assertTrue("Findings saved for test monitor", findings[1].relatedDocIds.contains("5"))
+        }"""
+        var httpResponse = adminClient().makeRequest(
+            "GET", "/${monitor.dataSources.queryIndex}/_search",
+            StringEntity(request, ContentType.APPLICATION_JSON)
+        )
+        assertEquals("Search failed", RestStatus.OK, httpResponse.restStatus())
+        var searchResponse = SearchResponse.fromXContent(createParser(JsonXContent.jsonXContent, httpResponse.entity.content))
+        searchResponse.hits.totalHits?.let { assertEquals("Query saved in query index", 0L, it.value) }
     }
 
-    fun `test execute monitor generates alerts and findings with per trigger execution for actions`() {
+    fun `test execute monitor returns search result with dryrun then without dryrun ensure dry run query not saved`() {
         val testIndex = createTestIndex()
         val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(MILLIS))
         val testDoc = """{
@@ -298,24 +348,13 @@ class DocumentMonitorRunnerIT : AlertingRestTestCase() {
         val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", name = "3", fields = listOf())
         val docLevelInput = DocLevelMonitorInput("description", listOf(testIndex), listOf(docQuery))
 
-        val actionExecutionScope = PerExecutionActionScope()
-        val actionExecutionPolicy = ActionExecutionPolicy(actionExecutionScope)
-        val actions = (0..randomInt(10)).map {
-            randomActionWithPolicy(
-                template = randomTemplateScript("Hello {{ctx.monitor.name}}"),
-                destinationId = createDestination().id,
-                actionExecutionPolicy = actionExecutionPolicy
-            )
-        }
-
-        val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN, actions = actions)
-        val monitor = createMonitor(randomDocumentLevelMonitor(inputs = listOf(docLevelInput), triggers = listOf(trigger)))
-        assertNotNull(monitor.id)
+        val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN)
+        val monitor = randomDocumentLevelMonitor(inputs = listOf(docLevelInput), triggers = listOf(trigger))
 
         indexDoc(testIndex, "1", testDoc)
-        indexDoc(testIndex, "5", testDoc)
+        indexDoc(testIndex, "2", testDoc)
 
-        val response = executeMonitor(monitor.id)
+        val response = executeMonitor(monitor, params = DRYRUN_MONITOR)
 
         val output = entityAsMap(response)
 
@@ -325,36 +364,79 @@ class DocumentMonitorRunnerIT : AlertingRestTestCase() {
         @Suppress("UNCHECKED_CAST")
         val matchingDocsToQuery = searchResult[docQuery.id] as List<String>
         assertEquals("Incorrect search result", 2, matchingDocsToQuery.size)
-        assertTrue("Incorrect search result", matchingDocsToQuery.containsAll(listOf("1|$testIndex", "5|$testIndex")))
+        assertTrue("Incorrect search result", matchingDocsToQuery.contains("1|$testIndex"))
+        assertTrue("Incorrect search result", matchingDocsToQuery.contains("2|$testIndex"))
 
-        for (triggerResult in output.objectMap("trigger_results").values) {
-            assertEquals(2, triggerResult.objectMap("action_results").values.size)
-            for (alertActionResult in triggerResult.objectMap("action_results").values) {
-                assertEquals(actions.size, alertActionResult.values.size)
-                for (actionResult in alertActionResult.values) {
-                    @Suppress("UNCHECKED_CAST") val actionOutput = (actionResult as Map<String, Map<String, String>>)["output"]
-                        as Map<String, String>
-                    assertEquals("Hello ${monitor.name}", actionOutput["subject"])
-                    assertEquals("Hello ${monitor.name}", actionOutput["message"])
-                }
+        // ensure doc level query is deleted on dry run
+        val request = """{
+            "size": 10,
+            "query": {
+                "match_all": {}
             }
-        }
+        }"""
+        var httpResponse = adminClient().makeRequest(
+            "GET", "/${monitor.dataSources.queryIndex}/_search",
+            StringEntity(request, ContentType.APPLICATION_JSON)
+        )
+        assertEquals("Search failed", RestStatus.OK, httpResponse.restStatus())
+        var searchResponse = SearchResponse.fromXContent(createParser(JsonXContent.jsonXContent, httpResponse.entity.content))
+        searchResponse.hits.totalHits?.let { assertEquals(0L, it.value) }
 
-        val alerts = searchAlertsWithFilter(monitor)
+        // create and execute second monitor not as dryrun
+        val testIndex2 = createTestIndex("test1")
+        val testTime2 = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(MILLIS))
+        val testDoc2 = """{
+            "message" : "This is an error from IAD region",
+            "test_strict_date_time" : "$testTime2",
+            "test_field" : "us-east-1"
+        }"""
+
+        val docQuery2 = DocLevelQuery(query = "test_field:\"us-east-1\"", name = "3", fields = listOf())
+        val docLevelInput2 = DocLevelMonitorInput("description", listOf(testIndex2), listOf(docQuery2))
+
+        val trigger2 = randomDocumentLevelTrigger(condition = ALWAYS_RUN)
+        val monitor2 = createMonitor(randomDocumentLevelMonitor(inputs = listOf(docLevelInput2), triggers = listOf(trigger2)))
+        assertNotNull(monitor2.id)
+
+        indexDoc(testIndex2, "1", testDoc2)
+        indexDoc(testIndex2, "5", testDoc2)
+
+        val response2 = executeMonitor(monitor2.id)
+        val output2 = entityAsMap(response2)
+
+        assertEquals(monitor2.name, output2["monitor_name"])
+        @Suppress("UNCHECKED_CAST")
+        val searchResult2 = (output2.objectMap("input_results")["results"] as List<Map<String, Any>>).first()
+        @Suppress("UNCHECKED_CAST")
+        val matchingDocsToQuery2 = searchResult2[docQuery2.id] as List<String>
+        assertEquals("Incorrect search result", 2, matchingDocsToQuery2.size)
+        assertTrue("Incorrect search result", matchingDocsToQuery2.containsAll(listOf("1|$testIndex2", "5|$testIndex2")))
+
+        val alerts = searchAlertsWithFilter(monitor2)
         assertEquals("Alert saved for test monitor", 2, alerts.size)
 
-        val findings = searchFindings(monitor)
+        val findings = searchFindings(monitor2)
         assertEquals("Findings saved for test monitor", 2, findings.size)
-        assertTrue("Findings saved for test monitor", findings[0].relatedDocIds.contains("1"))
-        assertTrue("Findings saved for test monitor", findings[1].relatedDocIds.contains("5"))
-    }
+        assertTrue("Findings saved for test monitor", findings[0].relatedDocIds.contains("5"))
+        assertTrue("Findings saved for test monitor", findings[1].relatedDocIds.contains("1"))
 
-    fun `test execute monitor with wildcard index that generates alerts and findings for EQUALS query operator`() {
-        val testIndexPrefix = "test-index-${randomAlphaOfLength(10).lowercase(Locale.ROOT)}"
-        val testQueryName = "wildcard-test-query"
-        val testIndex = createTestIndex("${testIndexPrefix}1")
-        val testIndex2 = createTestIndex("${testIndexPrefix}2")
+        // ensure query from second monitor was saved
+        val expectedQueries = listOf("test_field_test1_${monitor2.id}:\"us-east-1\"")
+        httpResponse = adminClient().makeRequest(
+            "GET", "/${monitor.dataSources.queryIndex}/_search",
+            StringEntity(request, ContentType.APPLICATION_JSON)
+        )
+        assertEquals("Search failed", RestStatus.OK, httpResponse.restStatus())
+        searchResponse = SearchResponse.fromXContent(createParser(JsonXContent.jsonXContent, httpResponse.entity.content))
+        searchResponse.hits.forEach { hit ->
+            val query = ((hit.sourceAsMap["query"] as Map<String, Any>)["query_string"] as Map<String, Any>)["query"]
+            assertTrue(expectedQueries.contains(query))
+        }
+        searchResponse.hits.totalHits?.let { assertEquals("Query saved in query index", 1L, it.value) }
+    }
 
+    fun `test execute monitor generates alerts and findings`() {
+        val testIndex = createTestIndex()
         val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(MILLIS))
         val testDoc = """{
             "message" : "This is an error from IAD region",
@@ -362,15 +444,15 @@ class DocumentMonitorRunnerIT : AlertingRestTestCase() {
             "test_field" : "us-west-2"
         }"""
 
-        val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", name = testQueryName, fields = listOf())
-        val docLevelInput = DocLevelMonitorInput("description", listOf("$testIndexPrefix*"), listOf(docQuery))
+        val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", name = "3", fields = listOf())
+        val docLevelInput = DocLevelMonitorInput("description", listOf(testIndex), listOf(docQuery))
 
-        val trigger = randomDocumentLevelTrigger(condition = Script("query[name=$testQueryName]"))
+        val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN)
         val monitor = createMonitor(randomDocumentLevelMonitor(inputs = listOf(docLevelInput), triggers = listOf(trigger)))
         assertNotNull(monitor.id)
 
         indexDoc(testIndex, "1", testDoc)
-        indexDoc(testIndex2, "5", testDoc)
+        indexDoc(testIndex, "5", testDoc)
 
         val response = executeMonitor(monitor.id)
 
@@ -382,23 +464,57 @@ class DocumentMonitorRunnerIT : AlertingRestTestCase() {
         @Suppress("UNCHECKED_CAST")
         val matchingDocsToQuery = searchResult[docQuery.id] as List<String>
         assertEquals("Incorrect search result", 2, matchingDocsToQuery.size)
-        assertTrue("Incorrect search result", matchingDocsToQuery.containsAll(listOf("1|$testIndex", "5|$testIndex2")))
+        assertTrue("Incorrect search result", matchingDocsToQuery.containsAll(listOf("1|$testIndex", "5|$testIndex")))
 
         val alerts = searchAlertsWithFilter(monitor)
         assertEquals("Alert saved for test monitor", 2, alerts.size)
 
         val findings = searchFindings(monitor)
         assertEquals("Findings saved for test monitor", 2, findings.size)
-        val foundFindings = findings.filter { it.relatedDocIds.contains("1") || it.relatedDocIds.contains("5") }
-        assertEquals("Didn't find findings for docs 1 and 5", 2, foundFindings.size)
+        assertTrue("Findings saved for test monitor", findings[0].relatedDocIds.contains("5"))
+        assertTrue("Findings saved for test monitor", findings[1].relatedDocIds.contains("1"))
     }
 
-    fun `test execute monitor with wildcard index that generates alerts and findings for NOT EQUALS query operator`() {
-        val testIndexPrefix = "test-index-${randomAlphaOfLength(10).lowercase(Locale.ROOT)}"
-        val testQueryName = "wildcard-test-query"
-        val testIndex = createTestIndex("${testIndexPrefix}1")
-        val testIndex2 = createTestIndex("${testIndexPrefix}2")
+    fun `test monitor run generates no error alerts with versionconflictengineexception with locks`() {
+        val testIndex = createTestIndex()
+        val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(MILLIS))
+        val testDoc = """{
+            "message" : "This is an error from IAD region",
+            "test_strict_date_time" : "$testTime",
+            "test_field" : "us-west-2"
+        }"""
+
+        val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", name = "3", fields = listOf())
+        val docLevelInput = DocLevelMonitorInput("description", listOf(testIndex), listOf(docQuery))
+
+        val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN)
+        val monitor = createMonitor(
+            randomDocumentLevelMonitor(
+                name = "__lag-monitor-test__",
+                inputs = listOf(docLevelInput),
+                triggers = listOf(trigger),
+                schedule = IntervalSchedule(interval = 1, unit = ChronoUnit.MINUTES)
+            )
+        )
+        assertNotNull(monitor.id)
+
+        indexDoc(testIndex, "1", testDoc)
+        indexDoc(testIndex, "5", testDoc)
+        Thread.sleep(240000)
+
+        val inputMap = HashMap<String, Any>()
+        inputMap["searchString"] = monitor.name
+
+        val responseMap = getAlerts(inputMap).asMap()
+        val alerts = (responseMap["alerts"] as ArrayList<Map<String, Any>>)
+        alerts.forEach {
+            assertTrue(it["error_message"] == null)
+        }
+    }
 
+    @AwaitsFix(bugUrl = "")
+    fun `test monitor run generate lock and monitor delete removes lock`() {
+        val testIndex = createTestIndex()
         val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(MILLIS))
         val testDoc = """{
             "message" : "This is an error from IAD region",
@@ -406,15 +522,57 @@ class DocumentMonitorRunnerIT : AlertingRestTestCase() {
             "test_field" : "us-west-2"
         }"""
 
-        val docQuery = DocLevelQuery(query = "NOT (test_field:\"us-west-1\")", name = testQueryName, fields = listOf())
-        val docLevelInput = DocLevelMonitorInput("description", listOf("$testIndexPrefix*"), listOf(docQuery))
+        val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", name = "3", fields = listOf())
+        val docLevelInput = DocLevelMonitorInput("description", listOf(testIndex), listOf(docQuery))
 
-        val trigger = randomDocumentLevelTrigger(condition = Script("query[name=$testQueryName]"))
+        val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN)
+        val monitor = createMonitor(
+            randomDocumentLevelMonitor(
+                inputs = listOf(docLevelInput),
+                triggers = listOf(trigger),
+                schedule = IntervalSchedule(interval = 1, unit = ChronoUnit.MINUTES)
+            )
+        )
+        assertNotNull(monitor.id)
+
+        indexDoc(testIndex, "1", testDoc)
+        indexDoc(testIndex, "5", testDoc)
+        OpenSearchTestCase.waitUntil({
+            val response = client().makeRequest("HEAD", LockService.LOCK_INDEX_NAME)
+            return@waitUntil (response.restStatus().status == 200)
+        }, 240, TimeUnit.SECONDS)
+
+        var response = client().makeRequest("GET", LockService.LOCK_INDEX_NAME + "/_search")
+        var responseMap = entityAsMap(response)
+        var noOfLocks = ((responseMap["hits"] as Map<String, Any>)["hits"] as List<Any>).size
+        assertEquals(1, noOfLocks)
+
+        deleteMonitor(monitor)
+        refreshIndex(LockService.LOCK_INDEX_NAME)
+        response = client().makeRequest("GET", LockService.LOCK_INDEX_NAME + "/_search")
+        responseMap = entityAsMap(response)
+        noOfLocks = ((responseMap["hits"] as Map<String, Any>)["hits"] as List<Any>).size
+        assertEquals(0, noOfLocks)
+    }
+
+    fun `test execute monitor with tag as trigger condition generates alerts and findings`() {
+        val testIndex = createTestIndex()
+        val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(MILLIS))
+        val testDoc = """{
+            "message" : "This is an error from IAD region",
+            "test_strict_date_time" : "$testTime",
+            "test_field" : "us-west-2"
+        }"""
+
+        val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", name = "3", tags = listOf("test_tag"), fields = listOf())
+        val docLevelInput = DocLevelMonitorInput("description", listOf(testIndex), listOf(docQuery))
+
+        val trigger = randomDocumentLevelTrigger(condition = Script("query[tag=test_tag]"))
         val monitor = createMonitor(randomDocumentLevelMonitor(inputs = listOf(docLevelInput), triggers = listOf(trigger)))
         assertNotNull(monitor.id)
 
         indexDoc(testIndex, "1", testDoc)
-        indexDoc(testIndex2, "5", testDoc)
+        indexDoc(testIndex, "5", testDoc)
 
         val response = executeMonitor(monitor.id)
 
@@ -426,20 +584,19 @@ class DocumentMonitorRunnerIT : AlertingRestTestCase() {
         @Suppress("UNCHECKED_CAST")
         val matchingDocsToQuery = searchResult[docQuery.id] as List<String>
         assertEquals("Incorrect search result", 2, matchingDocsToQuery.size)
-        assertTrue("Incorrect search result", matchingDocsToQuery.containsAll(listOf("1|$testIndex", "5|$testIndex2")))
+        assertTrue("Incorrect search result", matchingDocsToQuery.containsAll(listOf("1|$testIndex", "5|$testIndex")))
 
         val alerts = searchAlertsWithFilter(monitor)
         assertEquals("Alert saved for test monitor", 2, alerts.size)
 
         val findings = searchFindings(monitor)
         assertEquals("Findings saved for test monitor", 2, findings.size)
-        val foundFindings = findings.filter { it.relatedDocIds.contains("1") || it.relatedDocIds.contains("5") }
-        assertEquals("Didn't find findings for docs 1 and 5", 2, foundFindings.size)
+        assertTrue("Findings saved for test monitor", findings[0].relatedDocIds.contains("5"))
+        assertTrue("Findings saved for test monitor", findings[1].relatedDocIds.contains("1"))
     }
 
-    fun `test execute monitor with new index added after first execution that generates alerts and findings`() {
-        val testIndex = createTestIndex("test1")
-        val testIndex2 = createTestIndex("test2")
+    fun `test execute monitor input error`() {
+        val testIndex = createTestIndex()
         val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(MILLIS))
         val testDoc = """{
             "message" : "This is an error from IAD region",
@@ -447,479 +604,257 @@ class DocumentMonitorRunnerIT : AlertingRestTestCase() {
             "test_field" : "us-west-2"
         }"""
 
-        val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", name = "3", fields = listOf())
-        val docLevelInput = DocLevelMonitorInput("description", listOf("test*"), listOf(docQuery))
+        val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", name = "3", tags = listOf("test_tag"), fields = listOf())
+        val docLevelInput = DocLevelMonitorInput("description", listOf(testIndex), listOf(docQuery))
 
         val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN)
         val monitor = createMonitor(randomDocumentLevelMonitor(inputs = listOf(docLevelInput), triggers = listOf(trigger)))
         assertNotNull(monitor.id)
 
-        indexDoc(testIndex, "1", testDoc)
-        indexDoc(testIndex2, "5", testDoc)
-        executeMonitor(monitor.id)
-
-        var alerts = searchAlertsWithFilter(monitor)
-        assertEquals("Alert saved for test monitor", 2, alerts.size)
-
-        var findings = searchFindings(monitor)
-        assertEquals("Findings saved for test monitor", 2, findings.size)
-
-        var foundFindings = findings.filter { it.relatedDocIds.contains("1") || it.relatedDocIds.contains("5") }
-        assertEquals("Findings saved for test monitor expected 1 and 5", 2, foundFindings.size)
-
-        // clear previous findings and alerts
-        deleteIndex(ALL_FINDING_INDEX_PATTERN)
-        deleteIndex(ALL_ALERT_INDEX_PATTERN)
-
-        val testIndex3 = createTestIndex("test3")
-        indexDoc(testIndex3, "10", testDoc)
-        indexDoc(testIndex, "14", testDoc)
-        indexDoc(testIndex2, "51", testDoc)
+        deleteIndex(testIndex)
 
         val response = executeMonitor(monitor.id)
 
         val output = entityAsMap(response)
-
         assertEquals(monitor.name, output["monitor_name"])
         @Suppress("UNCHECKED_CAST")
-        val searchResult = (output.objectMap("input_results")["results"] as List<Map<String, Any>>).first()
-        @Suppress("UNCHECKED_CAST")
-        val matchingDocsToQuery = searchResult[docQuery.id] as List<String>
-        assertEquals("Incorrect search result", 3, matchingDocsToQuery.size)
-        assertTrue("Incorrect search result", matchingDocsToQuery.containsAll(listOf("14|$testIndex", "51|$testIndex2", "10|$testIndex3")))
-
-        alerts = searchAlertsWithFilter(monitor)
-        assertEquals("Alert saved for test monitor", 3, alerts.size)
-
-        findings = searchFindings(monitor)
-        assertEquals("Findings saved for test monitor", 3, findings.size)
+        val inputResults = output.stringMap("input_results")
+        assertTrue("Missing monitor error message", (inputResults?.get("error") as String).isNotEmpty())
 
-        foundFindings = findings.filter {
-            it.relatedDocIds.contains("14") || it.relatedDocIds.contains("51") || it.relatedDocIds.contains("10")
-        }
-        assertEquals("Findings saved for test monitor expected 14, 51 and 10", 3, foundFindings.size)
+        val alerts = searchAlerts(monitor)
+        assertEquals("Alert not saved", 1, alerts.size)
+        assertEquals("Alert status is incorrect", Alert.State.ERROR, alerts[0].state)
     }
 
-    fun `test execute monitor with indices having fields with same name but different data types`() {
-        val testIndex = createTestIndex(
-            "test1",
-            """"properties": {
-                    "source.device.port": { "type": "long" },
-                    "source.device.hwd.id": { "type": "long" },
-                    "nested_field": {
-                      "type": "nested",
-                      "properties": {
-                        "test1": {
-                          "type": "keyword"
-                        }
-                      }
-                    },
-                    "my_join_field": { 
-                      "type": "join",
-                      "relations": {
-                         "question": "answer" 
-                      }
-                   },
-                   "test_field" : { "type" : "integer" }
-                }
-            """.trimIndent()
-        )
-        var testDoc = """{
-            "source" : { "device": {"port" : 12345 } },
-            "nested_field": { "test1": "some text" },
-            "test_field": 12345
+    fun `test execute monitor generates alerts and findings with per alert execution for actions`() {
+        val testIndex = createTestIndex()
+        val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(MILLIS))
+        val testDoc = """{
+            "message" : "This is an error from IAD region",
+            "test_strict_date_time" : "$testTime",
+            "test_field" : "us-west-2"
         }"""
 
-        val docQuery1 = DocLevelQuery(
-            query = "(source.device.port:12345 AND test_field:12345) OR source.device.hwd.id:12345",
-            name = "4",
-            fields = listOf()
-        )
-        val docQuery2 = DocLevelQuery(
-            query = "(source.device.port:\"12345\" AND test_field:\"12345\") OR source.device.hwd.id:\"12345\"",
-            name = "5",
-            fields = listOf()
+        val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", name = "3", fields = listOf())
+        val docLevelInput = DocLevelMonitorInput("description", listOf(testIndex), listOf(docQuery))
+
+        val alertCategories = AlertCategory.values()
+        val actionExecutionScope = PerAlertActionScope(
+            actionableAlerts = (1..randomInt(alertCategories.size)).map { alertCategories[it - 1] }.toSet()
         )
-        val docLevelInput = DocLevelMonitorInput("description", listOf("test*"), listOf(docQuery1, docQuery2))
+        val actionExecutionPolicy = ActionExecutionPolicy(actionExecutionScope)
+        val actions = (0..randomInt(10)).map {
+            randomActionWithPolicy(
+                template = randomTemplateScript("Hello {{ctx.monitor.name}}"),
+                destinationId = createDestination().id,
+                actionExecutionPolicy = actionExecutionPolicy
+            )
+        }
 
-        val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN)
+        val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN, actions = actions)
         val monitor = createMonitor(randomDocumentLevelMonitor(inputs = listOf(docLevelInput), triggers = listOf(trigger)))
         assertNotNull(monitor.id)
 
         indexDoc(testIndex, "1", testDoc)
-        executeMonitor(monitor.id)
-
-        var alerts = searchAlertsWithFilter(monitor)
-        assertEquals("Alert saved for test monitor", 1, alerts.size)
+        indexDoc(testIndex, "5", testDoc)
 
-        var findings = searchFindings(monitor)
-        assertEquals("Findings saved for test monitor", 1, findings.size)
+        val response = executeMonitor(monitor.id)
 
-        // clear previous findings and alerts
-        deleteIndex(ALL_FINDING_INDEX_PATTERN)
-        deleteIndex(ALL_ALERT_INDEX_PATTERN)
+        val output = entityAsMap(response)
 
-        indexDoc(testIndex, "2", testDoc)
-
-        // no fields expanded as only index test1 is present
-        val oldExpectedQueries = listOf(
-            "(source.device.port_test__${monitor.id}:12345 AND test_field_test__${monitor.id}:12345) OR " +
-                "source.device.hwd.id_test__${monitor.id}:12345",
-            "(source.device.port_test__${monitor.id}:\"12345\" AND test_field_test__${monitor.id}:\"12345\") " +
-                "OR source.device.hwd.id_test__${monitor.id}:\"12345\""
-        )
+        assertEquals(monitor.name, output["monitor_name"])
+        @Suppress("UNCHECKED_CAST")
+        val searchResult = (output.objectMap("input_results")["results"] as List<Map<String, Any>>).first()
+        @Suppress("UNCHECKED_CAST")
+        val matchingDocsToQuery = searchResult[docQuery.id] as List<String>
+        assertEquals("Incorrect search result", 2, matchingDocsToQuery.size)
+        assertTrue("Incorrect search result", matchingDocsToQuery.containsAll(listOf("1|$testIndex", "5|$testIndex")))
 
-        val request = """{
-            "size": 10,
-            "query": {
-                "match_all": {}
+        for (triggerResult in output.objectMap("trigger_results").values) {
+            assertEquals(2, triggerResult.objectMap("action_results").values.size)
+            for (alertActionResult in triggerResult.objectMap("action_results").values) {
+                assertEquals(actions.size, alertActionResult.values.size)
+                for (actionResult in alertActionResult.values) {
+                    @Suppress("UNCHECKED_CAST") val actionOutput = (actionResult as Map<String, Map<String, String>>)["output"]
+                        as Map<String, String>
+                    assertEquals("Hello ${monitor.name}", actionOutput["subject"])
+                    assertEquals("Hello ${monitor.name}", actionOutput["message"])
+                }
             }
-        }"""
-        var httpResponse = adminClient().makeRequest(
-            "GET", "/${monitor.dataSources.queryIndex}/_search",
-            StringEntity(request, ContentType.APPLICATION_JSON)
-        )
-        assertEquals("Search failed", RestStatus.OK, httpResponse.restStatus())
-        var searchResponse = SearchResponse.fromXContent(createParser(JsonXContent.jsonXContent, httpResponse.entity.content))
-        searchResponse.hits.forEach { hit ->
-            val query = ((hit.sourceAsMap["query"] as Map<String, Any>)["query_string"] as Map<String, Any>)["query"]
-            assertTrue(oldExpectedQueries.contains(query))
         }
 
-        val testIndex2 = createTestIndex(
-            "test2",
-            """
-                "properties" : {
-                  "test_strict_date_time" : { "type" : "date", "format" : "strict_date_time" },
-                  "test_field" : { "type" : "keyword" },
-                  "number" : { "type" : "keyword" }
-                }
-            """.trimIndent()
-        )
-        testDoc = """{
-            "source" : { "device": {"port" : "12345" } },
-            "nested_field": { "test1": "some text" },
-            "test_field": "12345"
-        }"""
-        indexDoc(testIndex2, "1", testDoc)
-        executeMonitor(monitor.id)
-
-        // only fields source.device.port & test_field is expanded as they have same name but different data types
-        // in indices test1 & test2
-        val newExpectedQueries = listOf(
-            "(source.device.port_test2_${monitor.id}:12345 AND test_field_test2_${monitor.id}:12345) " +
-                "OR source.device.hwd.id_test__${monitor.id}:12345",
-            "(source.device.port_test1_${monitor.id}:12345 AND test_field_test1_${monitor.id}:12345) " +
-                "OR source.device.hwd.id_test__${monitor.id}:12345",
-            "(source.device.port_test2_${monitor.id}:\"12345\" AND test_field_test2_${monitor.id}:\"12345\") " +
-                "OR source.device.hwd.id_test__${monitor.id}:\"12345\"",
-            "(source.device.port_test1_${monitor.id}:\"12345\" AND test_field_test1_${monitor.id}:\"12345\") " +
-                "OR source.device.hwd.id_test__${monitor.id}:\"12345\""
-        )
+        refreshAllIndices()
 
-        alerts = searchAlertsWithFilter(monitor)
+        val alerts = searchAlertsWithFilter(monitor)
         assertEquals("Alert saved for test monitor", 2, alerts.size)
 
-        findings = searchFindings(monitor)
+        val findings = searchFindings(monitor)
         assertEquals("Findings saved for test monitor", 2, findings.size)
-
-        httpResponse = adminClient().makeRequest(
-            "GET", "/${monitor.dataSources.queryIndex}/_search",
-            StringEntity(request, ContentType.APPLICATION_JSON)
-        )
-        assertEquals("Search failed", RestStatus.OK, httpResponse.restStatus())
-        searchResponse = SearchResponse.fromXContent(createParser(JsonXContent.jsonXContent, httpResponse.entity.content))
-        searchResponse.hits.forEach { hit ->
-            val query = ((hit.sourceAsMap["query"] as Map<String, Any>)["query_string"] as Map<String, Any>)["query"]
-            assertTrue(oldExpectedQueries.contains(query) || newExpectedQueries.contains(query))
-        }
+        assertTrue("Findings saved for test monitor", findings[0].relatedDocIds.contains("5"))
+        assertTrue("Findings saved for test monitor", findings[1].relatedDocIds.contains("1"))
     }
 
-    fun `test execute monitor with indices having fields with same name but with different nesting`() {
-        val testIndex = createTestIndex(
-            "test1",
-            """"properties": {
-                    "nested_field": {
-                      "type": "nested",
-                      "properties": {
-                        "test1": {
-                          "type": "keyword"
-                        }
-                      }
-                    }
-                }
-            """.trimIndent()
-        )
-
-        val testIndex2 = createTestIndex(
-            "test2",
-            """"properties": {
-                      "nested_field": {
-                          "properties": {
-                            "test1": {
-                              "type": "keyword"
-                            }
-                          }
-                        }
-                    }
-            """.trimIndent()
-        )
+    fun `test execute monitor generates alerts and findings with per trigger execution for actions`() {
+        val testIndex = createTestIndex()
+        val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(MILLIS))
         val testDoc = """{
-            "nested_field": { "test1": "12345" }
+            "message" : "This is an error from IAD region",
+            "test_strict_date_time" : "$testTime",
+            "test_field" : "us-west-2"
         }"""
 
-        val docQuery = DocLevelQuery(
-            query = "nested_field.test1:\"12345\"",
-            name = "5",
-            fields = listOf()
-        )
-        val docLevelInput = DocLevelMonitorInput("description", listOf("test*"), listOf(docQuery))
+        val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", name = "3", fields = listOf())
+        val docLevelInput = DocLevelMonitorInput("description", listOf(testIndex), listOf(docQuery))
 
-        val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN)
+        val actionExecutionScope = PerExecutionActionScope()
+        val actionExecutionPolicy = ActionExecutionPolicy(actionExecutionScope)
+        val actions = (0..randomInt(10)).map {
+            randomActionWithPolicy(
+                template = randomTemplateScript("Hello {{ctx.monitor.name}}"),
+                destinationId = createDestination().id,
+                actionExecutionPolicy = actionExecutionPolicy
+            )
+        }
+
+        val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN, actions = actions)
         val monitor = createMonitor(randomDocumentLevelMonitor(inputs = listOf(docLevelInput), triggers = listOf(trigger)))
         assertNotNull(monitor.id)
 
         indexDoc(testIndex, "1", testDoc)
-        indexDoc(testIndex2, "1", testDoc)
+        indexDoc(testIndex, "5", testDoc)
 
-        executeMonitor(monitor.id)
+        val response = executeMonitor(monitor.id)
+
+        val output = entityAsMap(response)
+
+        assertEquals(monitor.name, output["monitor_name"])
+        @Suppress("UNCHECKED_CAST")
+        val searchResult = (output.objectMap("input_results")["results"] as List<Map<String, Any>>).first()
+        @Suppress("UNCHECKED_CAST")
+        val matchingDocsToQuery = searchResult[docQuery.id] as List<String>
+        assertEquals("Incorrect search result", 2, matchingDocsToQuery.size)
+        assertTrue("Incorrect search result", matchingDocsToQuery.containsAll(listOf("1|$testIndex", "5|$testIndex")))
+
+        for (triggerResult in output.objectMap("trigger_results").values) {
+            assertEquals(2, triggerResult.objectMap("action_results").values.size)
+            for (alertActionResult in triggerResult.objectMap("action_results").values) {
+                assertEquals(actions.size, alertActionResult.values.size)
+                for (actionResult in alertActionResult.values) {
+                    @Suppress("UNCHECKED_CAST") val actionOutput = (actionResult as Map<String, Map<String, String>>)["output"]
+                        as Map<String, String>
+                    assertEquals("Hello ${monitor.name}", actionOutput["subject"])
+                    assertEquals("Hello ${monitor.name}", actionOutput["message"])
+                }
+            }
+        }
 
         val alerts = searchAlertsWithFilter(monitor)
         assertEquals("Alert saved for test monitor", 2, alerts.size)
 
         val findings = searchFindings(monitor)
         assertEquals("Findings saved for test monitor", 2, findings.size)
-
-        // as mappings of source.id & test_field are different so, both of them expands
-        val expectedQueries = listOf(
-            "nested_field.test1_test__${monitor.id}:\"12345\""
-        )
-
-        val request = """{
-            "size": 10,
-            "query": {
-                "match_all": {}
-            }
-        }"""
-        var httpResponse = adminClient().makeRequest(
-            "GET", "/${monitor.dataSources.queryIndex}/_search",
-            StringEntity(request, ContentType.APPLICATION_JSON)
-        )
-        assertEquals("Search failed", RestStatus.OK, httpResponse.restStatus())
-        var searchResponse = SearchResponse.fromXContent(createParser(JsonXContent.jsonXContent, httpResponse.entity.content))
-        searchResponse.hits.forEach { hit ->
-            val query = ((hit.sourceAsMap["query"] as Map<String, Any>)["query_string"] as Map<String, Any>)["query"]
-            assertTrue(expectedQueries.contains(query))
-        }
+        assertTrue("Findings saved for test monitor", findings[0].relatedDocIds.contains("5"))
+        assertTrue("Findings saved for test monitor", findings[1].relatedDocIds.contains("1"))
     }
 
-    fun `test execute monitor with indices having fields with same name but different field mappings`() {
-        val testIndex = createTestIndex(
-            "test1",
-            """"properties": {
-                    "source": {
-                        "properties": {
-                            "id": {
-                                "type":"text",
-                                "analyzer":"whitespace" 
-                            }
-                        }
-                    },
-                   "test_field" : {
-                        "type":"text",
-                        "analyzer":"whitespace"
-                    }
-                }
-            """.trimIndent()
-        )
+    fun `test execute monitor with wildcard index that generates alerts and findings for EQUALS query operator`() {
+        val testIndexPrefix = "test-index-${randomAlphaOfLength(10).lowercase(Locale.ROOT)}"
+        val testQueryName = "wildcard-test-query"
+        val testIndex = createTestIndex("${testIndexPrefix}1")
+        val testIndex2 = createTestIndex("${testIndexPrefix}2")
 
-        val testIndex2 = createTestIndex(
-            "test2",
-            """"properties": {
-                    "source": {
-                        "properties": {
-                            "id": {
-                                "type":"text"
-                            }
-                        }
-                    },
-                   "test_field" : {
-                        "type":"text"
-                    }
-                }
-            """.trimIndent()
-        )
+        val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(MILLIS))
         val testDoc = """{
-            "source" : {"id" : "12345" },
-            "nested_field": { "test1": "some text" },
-            "test_field": "12345"
+            "message" : "This is an error from IAD region",
+            "test_strict_date_time" : "$testTime",
+            "test_field" : "us-west-2"
         }"""
 
-        val docQuery = DocLevelQuery(
-            query = "test_field:\"12345\" AND source.id:\"12345\"",
-            name = "5",
-            fields = listOf()
-        )
-        val docLevelInput = DocLevelMonitorInput("description", listOf("test*"), listOf(docQuery))
+        val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", name = testQueryName, fields = listOf())
+        val docLevelInput = DocLevelMonitorInput("description", listOf("$testIndexPrefix*"), listOf(docQuery))
 
-        val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN)
+        val trigger = randomDocumentLevelTrigger(condition = Script("query[name=$testQueryName]"))
         val monitor = createMonitor(randomDocumentLevelMonitor(inputs = listOf(docLevelInput), triggers = listOf(trigger)))
         assertNotNull(monitor.id)
 
         indexDoc(testIndex, "1", testDoc)
-        indexDoc(testIndex2, "1", testDoc)
+        indexDoc(testIndex2, "5", testDoc)
 
-        executeMonitor(monitor.id)
+        val response = executeMonitor(monitor.id)
 
-        val alerts = searchAlertsWithFilter(monitor)
-        assertEquals("Alert saved for test monitor", 2, alerts.size)
+        val output = entityAsMap(response)
 
-        val findings = searchFindings(monitor)
-        assertEquals("Findings saved for test monitor", 2, findings.size)
+        assertEquals(monitor.name, output["monitor_name"])
+        @Suppress("UNCHECKED_CAST")
+        val searchResult = (output.objectMap("input_results")["results"] as List<Map<String, Any>>).first()
+        @Suppress("UNCHECKED_CAST")
+        val matchingDocsToQuery = searchResult[docQuery.id] as List<String>
+        assertEquals("Incorrect search result", 2, matchingDocsToQuery.size)
+        assertTrue("Incorrect search result", matchingDocsToQuery.containsAll(listOf("1|$testIndex", "5|$testIndex2")))
 
-        // as mappings of source.id & test_field are different so, both of them expands
-        val expectedQueries = listOf(
-            "test_field_test2_${monitor.id}:\"12345\" AND source.id_test2_${monitor.id}:\"12345\"",
-            "test_field_test1_${monitor.id}:\"12345\" AND source.id_test1_${monitor.id}:\"12345\""
-        )
+        val alerts = searchAlertsWithFilter(monitor)
+        assertEquals("Alert saved for test monitor", 2, alerts.size)
 
-        val request = """{
-            "size": 10,
-            "query": {
-                "match_all": {}
-            }
-        }"""
-        var httpResponse = adminClient().makeRequest(
-            "GET", "/${monitor.dataSources.queryIndex}/_search",
-            StringEntity(request, ContentType.APPLICATION_JSON)
-        )
-        assertEquals("Search failed", RestStatus.OK, httpResponse.restStatus())
-        var searchResponse = SearchResponse.fromXContent(createParser(JsonXContent.jsonXContent, httpResponse.entity.content))
-        searchResponse.hits.forEach { hit ->
-            val query = ((hit.sourceAsMap["query"] as Map<String, Any>)["query_string"] as Map<String, Any>)["query"]
-            assertTrue(expectedQueries.contains(query))
-        }
+        val findings = searchFindings(monitor)
+        assertEquals("Findings saved for test monitor", 2, findings.size)
+        val foundFindings = findings.filter { it.relatedDocIds.contains("1") || it.relatedDocIds.contains("5") }
+        assertEquals("Didn't find findings for docs 1 and 5", 2, foundFindings.size)
     }
 
-    fun `test execute monitor with indices having fields with same name but different field mappings in multiple indices`() {
-        val testIndex = createTestIndex(
-            "test1",
-            """"properties": {
-                    "source": {
-                        "properties": {
-                            "device": {
-                                "properties": {
-                                    "hwd": {
-                                        "properties": {
-                                            "id": {
-                                                "type":"text",
-                                                "analyzer":"whitespace" 
-                                            }
-                                        }
-                                    } 
-                                }
-                            }
-                        }
-                    },
-                   "test_field" : {
-                        "type":"text" 
-                    }
-                }
-            """.trimIndent()
-        )
-
-        val testIndex2 = createTestIndex(
-            "test2",
-            """"properties": {
-                    "test_field" : {
-                        "type":"keyword"
-                    }
-                }
-            """.trimIndent()
-        )
-
-        val testIndex4 = createTestIndex(
-            "test4",
-            """"properties": {
-                   "source": {
-                        "properties": {
-                            "device": {
-                                "properties": {
-                                    "hwd": {
-                                        "properties": {
-                                            "id": {
-                                                "type":"text"
-                                            }
-                                        }
-                                    } 
-                                }
-                            }
-                        }
-                    },
-                   "test_field" : {
-                        "type":"text" 
-                    }
-                }
-            """.trimIndent()
-        )
+    fun `test execute monitor for bulk index findings`() {
+        val testIndexPrefix = "test-index-${randomAlphaOfLength(10).lowercase(Locale.ROOT)}"
+        val testQueryName = "wildcard-test-query"
+        val testIndex = createTestIndex("${testIndexPrefix}1")
+        val testIndex2 = createTestIndex("${testIndexPrefix}2")
 
-        val testDoc1 = """{
-            "source" : {"device" : {"hwd" : {"id" : "12345"}} },
-            "nested_field": { "test1": "some text" }
-        }"""
-        val testDoc2 = """{
-            "nested_field": { "test1": "some text" },
-            "test_field": "12345"
+        val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(MILLIS))
+        val testDoc = """{
+            "message" : "This is an error from IAD region",
+            "test_strict_date_time" : "$testTime",
+            "test_field" : "us-west-2"
         }"""
 
-        val docQuery1 = DocLevelQuery(
-            query = "test_field:\"12345\"",
-            name = "4",
-            fields = listOf()
-        )
-        val docQuery2 = DocLevelQuery(
-            query = "source.device.hwd.id:\"12345\"",
-            name = "5",
-            fields = listOf()
-        )
-
-        val docLevelInput = DocLevelMonitorInput("description", listOf("test*"), listOf(docQuery1, docQuery2))
+        val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", name = testQueryName, fields = listOf())
+        val docLevelInput = DocLevelMonitorInput("description", listOf("$testIndexPrefix*"), listOf(docQuery))
 
-        val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN)
+        val trigger = randomDocumentLevelTrigger(condition = Script("query[name=$testQueryName]"))
         val monitor = createMonitor(randomDocumentLevelMonitor(inputs = listOf(docLevelInput), triggers = listOf(trigger)))
         assertNotNull(monitor.id)
 
-        indexDoc(testIndex4, "1", testDoc1)
-        indexDoc(testIndex2, "1", testDoc2)
-        indexDoc(testIndex, "1", testDoc1)
-        indexDoc(testIndex, "2", testDoc2)
+        for (i in 0 until 9) {
+            indexDoc(testIndex, i.toString(), testDoc)
+        }
+        indexDoc(testIndex2, "3", testDoc)
+        adminClient().updateSettings("plugins.alerting.alert_findings_indexing_batch_size", 2)
 
-        executeMonitor(monitor.id)
+        val response = executeMonitor(monitor.id)
 
-        val alerts = searchAlertsWithFilter(monitor)
-        assertEquals("Alert saved for test monitor", 4, alerts.size)
+        val output = entityAsMap(response)
 
-        val findings = searchFindings(monitor)
-        assertEquals("Findings saved for test monitor", 4, findings.size)
+        assertEquals(monitor.name, output["monitor_name"])
+        @Suppress("UNCHECKED_CAST")
+        val searchResult = (output.objectMap("input_results")["results"] as List<Map<String, Any>>).first()
+        @Suppress("UNCHECKED_CAST")
+        val matchingDocsToQuery = searchResult[docQuery.id] as List<String>
+        assertEquals("Correct search result", 10, matchingDocsToQuery.size)
+        assertTrue("Correct search result", matchingDocsToQuery.containsAll(listOf("1|$testIndex", "2|$testIndex", "3|$testIndex2")))
 
-        val request = """{
-            "size": 0,
-            "query": {
-                "match_all": {}
-            }
-        }"""
-        val httpResponse = adminClient().makeRequest(
-            "GET", "/${monitor.dataSources.queryIndex}/_search",
-            StringEntity(request, ContentType.APPLICATION_JSON)
-        )
-        assertEquals("Search failed", RestStatus.OK, httpResponse.restStatus())
+        val alerts = searchAlertsWithFilter(monitor)
+        assertEquals("Alert saved for test monitor", 10, alerts.size)
 
-        val searchResponse = SearchResponse.fromXContent(createParser(JsonXContent.jsonXContent, httpResponse.entity.content))
-        searchResponse.hits.totalHits?.let { assertEquals(5L, it.value) }
+        val findings = searchFindings(monitor)
+        assertEquals("Findings saved for test monitor", 10, findings.size)
+        val foundFindings =
+            findings.filter { it.relatedDocIds.contains("1") || it.relatedDocIds.contains("2") || it.relatedDocIds.contains("3") }
+        assertEquals("Found findings for all docs", 4, foundFindings.size)
     }
 
-    fun `test no of queries generated for document-level monitor based on wildcard indexes`() {
-        val testIndex = createTestIndex("test1")
+    fun `test execute monitor with wildcard index that generates alerts and findings for NOT EQUALS query operator`() {
+        val testIndexPrefix = "test-index-${randomAlphaOfLength(10).lowercase(Locale.ROOT)}"
+        val testQueryName = "wildcard-test-query"
+        val testIndex = createTestIndex("${testIndexPrefix}1")
+        val testIndex2 = createTestIndex("${testIndexPrefix}2")
+
         val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(MILLIS))
         val testDoc = """{
             "message" : "This is an error from IAD region",
@@ -927,46 +862,38 @@ class DocumentMonitorRunnerIT : AlertingRestTestCase() {
             "test_field" : "us-west-2"
         }"""
 
-        val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", name = "3", fields = listOf())
-        val docLevelInput = DocLevelMonitorInput("description", listOf("test*"), listOf(docQuery))
+        val docQuery = DocLevelQuery(query = "NOT (test_field:\"us-west-1\")", name = testQueryName, fields = listOf())
+        val docLevelInput = DocLevelMonitorInput("description", listOf("$testIndexPrefix*"), listOf(docQuery))
 
-        val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN)
+        val trigger = randomDocumentLevelTrigger(condition = Script("query[name=$testQueryName]"))
         val monitor = createMonitor(randomDocumentLevelMonitor(inputs = listOf(docLevelInput), triggers = listOf(trigger)))
         assertNotNull(monitor.id)
 
         indexDoc(testIndex, "1", testDoc)
-        executeMonitor(monitor.id)
+        indexDoc(testIndex2, "5", testDoc)
 
-        val request = """{
-            "size": 0,
-            "query": {
-                "match_all": {}
-            }
-        }"""
-        var httpResponse = adminClient().makeRequest(
-            "GET", "/${monitor.dataSources.queryIndex}/_search",
-            StringEntity(request, ContentType.APPLICATION_JSON)
-        )
-        assertEquals("Search failed", RestStatus.OK, httpResponse.restStatus())
+        val response = executeMonitor(monitor.id)
 
-        var searchResponse = SearchResponse.fromXContent(createParser(JsonXContent.jsonXContent, httpResponse.entity.content))
-        searchResponse.hits.totalHits?.let { assertEquals(1L, it.value) }
+        val output = entityAsMap(response)
 
-        val testIndex2 = createTestIndex("test2")
-        indexDoc(testIndex2, "1", testDoc)
-        executeMonitor(monitor.id)
+        assertEquals(monitor.name, output["monitor_name"])
+        @Suppress("UNCHECKED_CAST")
+        val searchResult = (output.objectMap("input_results")["results"] as List<Map<String, Any>>).first()
+        @Suppress("UNCHECKED_CAST")
+        val matchingDocsToQuery = searchResult[docQuery.id] as List<String>
+        assertEquals("Incorrect search result", 2, matchingDocsToQuery.size)
+        assertTrue("Incorrect search result", matchingDocsToQuery.containsAll(listOf("1|$testIndex", "5|$testIndex2")))
 
-        httpResponse = adminClient().makeRequest(
-            "GET", "/${monitor.dataSources.queryIndex}/_search",
-            StringEntity(request, ContentType.APPLICATION_JSON)
-        )
-        assertEquals("Search failed", RestStatus.OK, httpResponse.restStatus())
+        val alerts = searchAlertsWithFilter(monitor)
+        assertEquals("Alert saved for test monitor", 2, alerts.size)
 
-        searchResponse = SearchResponse.fromXContent(createParser(JsonXContent.jsonXContent, httpResponse.entity.content))
-        searchResponse.hits.totalHits?.let { assertEquals(1L, it.value) }
+        val findings = searchFindings(monitor)
+        assertEquals("Findings saved for test monitor", 2, findings.size)
+        val foundFindings = findings.filter { it.relatedDocIds.contains("1") || it.relatedDocIds.contains("5") }
+        assertEquals("Didn't find findings for docs 1 and 5", 2, foundFindings.size)
     }
 
-    fun `test execute monitor with new index added after first execution that generates alerts and findings from new query`() {
+    fun `test execute monitor with new index added after first execution that generates alerts and findings`() {
         val testIndex = createTestIndex("test1")
         val testIndex2 = createTestIndex("test2")
         val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(MILLIS))
@@ -976,9 +903,8 @@ class DocumentMonitorRunnerIT : AlertingRestTestCase() {
             "test_field" : "us-west-2"
         }"""
 
-        val docQuery1 = DocLevelQuery(query = "test_field:\"us-west-2\"", name = "3", fields = listOf())
-        val docQuery2 = DocLevelQuery(query = "test_field_new:\"us-west-2\"", name = "4", fields = listOf())
-        val docLevelInput = DocLevelMonitorInput("description", listOf("test*"), listOf(docQuery1, docQuery2))
+        val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", name = "3", fields = listOf())
+        val docLevelInput = DocLevelMonitorInput("description", listOf("test*"), listOf(docQuery))
 
         val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN)
         val monitor = createMonitor(randomDocumentLevelMonitor(inputs = listOf(docLevelInput), triggers = listOf(trigger)))
@@ -1001,14 +927,10 @@ class DocumentMonitorRunnerIT : AlertingRestTestCase() {
         deleteIndex(ALL_FINDING_INDEX_PATTERN)
         deleteIndex(ALL_ALERT_INDEX_PATTERN)
 
-        val testDocNew = """{
-            "message" : "This is an error from IAD region",
-            "test_strict_date_time" : "$testTime",
-            "test_field_new" : "us-west-2"
-        }"""
-
         val testIndex3 = createTestIndex("test3")
-        indexDoc(testIndex3, "10", testDocNew)
+        indexDoc(testIndex3, "10", testDoc)
+        indexDoc(testIndex, "14", testDoc)
+        indexDoc(testIndex2, "51", testDoc)
 
         val response = executeMonitor(monitor.id)
 
@@ -1018,304 +940,1490 @@ class DocumentMonitorRunnerIT : AlertingRestTestCase() {
         @Suppress("UNCHECKED_CAST")
         val searchResult = (output.objectMap("input_results")["results"] as List<Map<String, Any>>).first()
         @Suppress("UNCHECKED_CAST")
-        val matchingDocsToQuery = searchResult[docQuery2.id] as List<String>
-        assertEquals("Incorrect search result", 1, matchingDocsToQuery.size)
-        assertTrue("Incorrect search result", matchingDocsToQuery.containsAll(listOf("10|$testIndex3")))
+        val matchingDocsToQuery = searchResult[docQuery.id] as List<String>
+        assertEquals("Incorrect search result", 3, matchingDocsToQuery.size)
+        assertTrue("Incorrect search result", matchingDocsToQuery.containsAll(listOf("14|$testIndex", "51|$testIndex2", "10|$testIndex3")))
 
         alerts = searchAlertsWithFilter(monitor)
-        assertEquals("Alert saved for test monitor", 1, alerts.size)
+        assertEquals("Alert saved for test monitor", 3, alerts.size)
 
         findings = searchFindings(monitor)
-        assertEquals("Findings saved for test monitor", 1, findings.size)
+        assertEquals("Findings saved for test monitor", 3, findings.size)
 
         foundFindings = findings.filter {
-            it.relatedDocIds.contains("10")
+            it.relatedDocIds.contains("14") || it.relatedDocIds.contains("51") || it.relatedDocIds.contains("10")
         }
-        assertEquals("Findings saved for test monitor expected 10", 1, foundFindings.size)
+        assertEquals("Findings saved for test monitor expected 14, 51 and 10", 3, foundFindings.size)
     }
 
-    fun `test document-level monitor when alias only has write index with 0 docs`() {
-        // Monitor should execute, but create 0 findings.
-        val alias = createTestAlias(includeWriteIndex = true)
-        val aliasIndex = alias.keys.first()
-        val query = randomDocLevelQuery(tags = listOf())
-        val input = randomDocLevelMonitorInput(indices = listOf(aliasIndex), queries = listOf(query))
-        val trigger = randomDocumentLevelTrigger(condition = Script("query[id=\"${query.id}\"]"))
-        val monitor = createMonitor(randomDocumentLevelMonitor(enabled = false, inputs = listOf(input), triggers = listOf(trigger)))
-
-        val response: Response
-        try {
-            response = executeMonitor(monitor.id)
+    fun `test execute monitor with indices having fields with same name but different data types`() {
+        val testIndex = createTestIndex(
+            "test1",
+            """"properties": {
+                    "source.device.port": { "type": "long" },
+                    "source.device.hwd.id": { "type": "long" },
+                    "nested_field": {
+                      "type": "nested",
+                      "properties": {
+                        "test1": {
+                          "type": "keyword"
+                        }
+                      }
+                    },
+                    "my_join_field": { 
+                      "type": "join",
+                      "relations": {
+                         "question": "answer" 
+                      }
+                   },
+                   "test_field" : { "type" : "integer" }
+                }
+            """.trimIndent()
+        )
+        var testDoc = """{
+            "source" : { "device": {"port" : 12345 } },
+            "nested_field": { "test1": "some text" },
+            "test_field": 12345
+        }"""
+
+        val docQuery1 = DocLevelQuery(
+            query = "(source.device.port:12345 AND test_field:12345) OR source.device.hwd.id:12345",
+            name = "4",
+            fields = listOf()
+        )
+        val docQuery2 = DocLevelQuery(
+            query = "(source.device.port:\"12345\" AND test_field:\"12345\") OR source.device.hwd.id:\"12345\"",
+            name = "5",
+            fields = listOf()
+        )
+        val docLevelInput = DocLevelMonitorInput("description", listOf("test*"), listOf(docQuery1, docQuery2))
+
+        val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN)
+        val monitor = createMonitor(randomDocumentLevelMonitor(inputs = listOf(docLevelInput), triggers = listOf(trigger)))
+        assertNotNull(monitor.id)
+
+        indexDoc(testIndex, "1", testDoc)
+        executeMonitor(monitor.id)
+
+        var alerts = searchAlertsWithFilter(monitor)
+        assertEquals("Alert saved for test monitor", 1, alerts.size)
+
+        var findings = searchFindings(monitor)
+        assertEquals("Findings saved for test monitor", 1, findings.size)
+
+        // clear previous findings and alerts
+        deleteIndex(ALL_FINDING_INDEX_PATTERN)
+        deleteIndex(ALL_ALERT_INDEX_PATTERN)
+
+        indexDoc(testIndex, "2", testDoc)
+
+        // no fields expanded as only index test1 is present
+        val oldExpectedQueries = listOf(
+            "(source.device.port_test__${monitor.id}:12345 AND test_field_test__${monitor.id}:12345) OR " +
+                "source.device.hwd.id_test__${monitor.id}:12345",
+            "(source.device.port_test__${monitor.id}:\"12345\" AND test_field_test__${monitor.id}:\"12345\") " +
+                "OR source.device.hwd.id_test__${monitor.id}:\"12345\""
+        )
+
+        val request = """{
+            "size": 10,
+            "query": {
+                "match_all": {}
+            }
+        }"""
+        var httpResponse = adminClient().makeRequest(
+            "GET", "/${monitor.dataSources.queryIndex}/_search",
+            StringEntity(request, ContentType.APPLICATION_JSON)
+        )
+        assertEquals("Search failed", RestStatus.OK, httpResponse.restStatus())
+        var searchResponse = SearchResponse.fromXContent(createParser(JsonXContent.jsonXContent, httpResponse.entity.content))
+        searchResponse.hits.forEach { hit ->
+            val query = ((hit.sourceAsMap["query"] as Map<String, Any>)["query_string"] as Map<String, Any>)["query"]
+            assertTrue(oldExpectedQueries.contains(query))
+        }
+
+        val testIndex2 = createTestIndex(
+            "test2",
+            """
+                "properties" : {
+                  "test_strict_date_time" : { "type" : "date", "format" : "strict_date_time" },
+                  "test_field" : { "type" : "keyword" },
+                  "number" : { "type" : "keyword" }
+                }
+            """.trimIndent()
+        )
+        testDoc = """{
+            "source" : { "device": {"port" : "12345" } },
+            "nested_field": { "test1": "some text" },
+            "test_field": "12345"
+        }"""
+        indexDoc(testIndex2, "1", testDoc)
+        executeMonitor(monitor.id)
+
+        // only fields source.device.port & test_field is expanded as they have same name but different data types
+        // in indices test1 & test2
+        val newExpectedQueries = listOf(
+            "(source.device.port_test2_${monitor.id}:12345 AND test_field_test2_${monitor.id}:12345) " +
+                "OR source.device.hwd.id_test__${monitor.id}:12345",
+            "(source.device.port_test1_${monitor.id}:12345 AND test_field_test1_${monitor.id}:12345) " +
+                "OR source.device.hwd.id_test__${monitor.id}:12345",
+            "(source.device.port_test2_${monitor.id}:\"12345\" AND test_field_test2_${monitor.id}:\"12345\") " +
+                "OR source.device.hwd.id_test__${monitor.id}:\"12345\"",
+            "(source.device.port_test1_${monitor.id}:\"12345\" AND test_field_test1_${monitor.id}:\"12345\") " +
+                "OR source.device.hwd.id_test__${monitor.id}:\"12345\""
+        )
+
+        alerts = searchAlertsWithFilter(monitor)
+        assertEquals("Alert saved for test monitor", 2, alerts.size)
+
+        findings = searchFindings(monitor)
+        assertEquals("Findings saved for test monitor", 2, findings.size)
+
+        httpResponse = adminClient().makeRequest(
+            "GET", "/${monitor.dataSources.queryIndex}/_search",
+            StringEntity(request, ContentType.APPLICATION_JSON)
+        )
+        assertEquals("Search failed", RestStatus.OK, httpResponse.restStatus())
+        searchResponse = SearchResponse.fromXContent(createParser(JsonXContent.jsonXContent, httpResponse.entity.content))
+        searchResponse.hits.forEach { hit ->
+            val query = ((hit.sourceAsMap["query"] as Map<String, Any>)["query_string"] as Map<String, Any>)["query"]
+            assertTrue(oldExpectedQueries.contains(query) || newExpectedQueries.contains(query))
+        }
+    }
+
+    fun `test execute monitor with indices having fields with same name but with different nesting`() {
+        val testIndex = createTestIndex(
+            "test1",
+            """"properties": {
+                    "nested_field": {
+                      "type": "nested",
+                      "properties": {
+                        "test1": {
+                          "type": "keyword"
+                        }
+                      }
+                    }
+                }
+            """.trimIndent()
+        )
+
+        val testIndex2 = createTestIndex(
+            "test2",
+            """"properties": {
+                      "nested_field": {
+                          "properties": {
+                            "test1": {
+                              "type": "keyword"
+                            }
+                          }
+                        }
+                    }
+            """.trimIndent()
+        )
+        val testDoc = """{
+            "nested_field": { "test1": "12345" }
+        }"""
+
+        val docQuery = DocLevelQuery(
+            query = "nested_field.test1:\"12345\"",
+            name = "5",
+            fields = listOf()
+        )
+        val docLevelInput = DocLevelMonitorInput("description", listOf("test*"), listOf(docQuery))
+
+        val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN)
+        val monitor = createMonitor(randomDocumentLevelMonitor(inputs = listOf(docLevelInput), triggers = listOf(trigger)))
+        assertNotNull(monitor.id)
+
+        indexDoc(testIndex, "1", testDoc)
+        indexDoc(testIndex2, "1", testDoc)
+
+        executeMonitor(monitor.id)
+
+        val alerts = searchAlertsWithFilter(monitor)
+        assertEquals("Alert saved for test monitor", 2, alerts.size)
+
+        val findings = searchFindings(monitor)
+        assertEquals("Findings saved for test monitor", 2, findings.size)
+
+        // as mappings of source.id & test_field are different so, both of them expands
+        val expectedQueries = listOf(
+            "nested_field.test1_test__${monitor.id}:\"12345\""
+        )
+
+        val request = """{
+            "size": 10,
+            "query": {
+                "match_all": {}
+            }
+        }"""
+        var httpResponse = adminClient().makeRequest(
+            "GET", "/${monitor.dataSources.queryIndex}/_search",
+            StringEntity(request, ContentType.APPLICATION_JSON)
+        )
+        assertEquals("Search failed", RestStatus.OK, httpResponse.restStatus())
+        var searchResponse = SearchResponse.fromXContent(createParser(JsonXContent.jsonXContent, httpResponse.entity.content))
+        searchResponse.hits.forEach { hit ->
+            val query = ((hit.sourceAsMap["query"] as Map<String, Any>)["query_string"] as Map<String, Any>)["query"]
+            assertTrue(expectedQueries.contains(query))
+        }
+    }
+
+    fun `test execute monitor with indices having fields with same name but different field mappings`() {
+        val testIndex = createTestIndex(
+            "test1",
+            """"properties": {
+                    "source": {
+                        "properties": {
+                            "id": {
+                                "type":"text",
+                                "analyzer":"whitespace" 
+                            }
+                        }
+                    },
+                   "test_field" : {
+                        "type":"text",
+                        "analyzer":"whitespace"
+                    }
+                }
+            """.trimIndent()
+        )
+
+        val testIndex2 = createTestIndex(
+            "test2",
+            """"properties": {
+                    "source": {
+                        "properties": {
+                            "id": {
+                                "type":"text"
+                            }
+                        }
+                    },
+                   "test_field" : {
+                        "type":"text"
+                    }
+                }
+            """.trimIndent()
+        )
+        val testDoc = """{
+            "source" : {"id" : "12345" },
+            "nested_field": { "test1": "some text" },
+            "test_field": "12345"
+        }"""
+
+        val docQuery = DocLevelQuery(
+            query = "test_field:\"12345\" AND source.id:\"12345\"",
+            name = "5",
+            fields = listOf()
+        )
+        val docLevelInput = DocLevelMonitorInput("description", listOf("test*"), listOf(docQuery))
+
+        val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN)
+        val monitor = createMonitor(randomDocumentLevelMonitor(inputs = listOf(docLevelInput), triggers = listOf(trigger)))
+        assertNotNull(monitor.id)
+
+        indexDoc(testIndex, "1", testDoc)
+        indexDoc(testIndex2, "1", testDoc)
+
+        executeMonitor(monitor.id)
+
+        val alerts = searchAlertsWithFilter(monitor)
+        assertEquals("Alert saved for test monitor", 2, alerts.size)
+
+        val findings = searchFindings(monitor)
+        assertEquals("Findings saved for test monitor", 2, findings.size)
+
+        // as mappings of source.id & test_field are different so, both of them expands
+        val expectedQueries = listOf(
+            "test_field_test2_${monitor.id}:\"12345\" AND source.id_test2_${monitor.id}:\"12345\"",
+            "test_field_test1_${monitor.id}:\"12345\" AND source.id_test1_${monitor.id}:\"12345\""
+        )
+
+        val request = """{
+            "size": 10,
+            "query": {
+                "match_all": {}
+            }
+        }"""
+        var httpResponse = adminClient().makeRequest(
+            "GET", "/${monitor.dataSources.queryIndex}/_search",
+            StringEntity(request, ContentType.APPLICATION_JSON)
+        )
+        assertEquals("Search failed", RestStatus.OK, httpResponse.restStatus())
+        var searchResponse = SearchResponse.fromXContent(createParser(JsonXContent.jsonXContent, httpResponse.entity.content))
+        searchResponse.hits.forEach { hit ->
+            val query = ((hit.sourceAsMap["query"] as Map<String, Any>)["query_string"] as Map<String, Any>)["query"]
+            assertTrue(expectedQueries.contains(query))
+        }
+    }
+
+    fun `test execute monitor with indices having fields with same name but different field mappings in multiple indices`() {
+        val testIndex = createTestIndex(
+            "test1",
+            """"properties": {
+                    "source": {
+                        "properties": {
+                            "device": {
+                                "properties": {
+                                    "hwd": {
+                                        "properties": {
+                                            "id": {
+                                                "type":"text",
+                                                "analyzer":"whitespace" 
+                                            }
+                                        }
+                                    } 
+                                }
+                            }
+                        }
+                    },
+                   "test_field" : {
+                        "type":"text" 
+                    }
+                }
+            """.trimIndent()
+        )
+
+        val testIndex2 = createTestIndex(
+            "test2",
+            """"properties": {
+                    "test_field" : {
+                        "type":"keyword"
+                    }
+                }
+            """.trimIndent()
+        )
+
+        val testIndex4 = createTestIndex(
+            "test4",
+            """"properties": {
+                   "source": {
+                        "properties": {
+                            "device": {
+                                "properties": {
+                                    "hwd": {
+                                        "properties": {
+                                            "id": {
+                                                "type":"text"
+                                            }
+                                        }
+                                    } 
+                                }
+                            }
+                        }
+                    },
+                   "test_field" : {
+                        "type":"text" 
+                    }
+                }
+            """.trimIndent()
+        )
+
+        val testDoc1 = """{
+            "source" : {"device" : {"hwd" : {"id" : "12345"}} },
+            "nested_field": { "test1": "some text" }
+        }"""
+        val testDoc2 = """{
+            "nested_field": { "test1": "some text" },
+            "test_field": "12345"
+        }"""
+
+        val docQuery1 = DocLevelQuery(
+            query = "test_field:\"12345\"",
+            name = "4",
+            fields = listOf()
+        )
+        val docQuery2 = DocLevelQuery(
+            query = "source.device.hwd.id:\"12345\"",
+            name = "5",
+            fields = listOf()
+        )
+
+        val docLevelInput = DocLevelMonitorInput("description", listOf("test*"), listOf(docQuery1, docQuery2))
+
+        val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN)
+        val monitor = createMonitor(randomDocumentLevelMonitor(inputs = listOf(docLevelInput), triggers = listOf(trigger)))
+        assertNotNull(monitor.id)
+
+        indexDoc(testIndex4, "1", testDoc1)
+        indexDoc(testIndex2, "1", testDoc2)
+        indexDoc(testIndex, "1", testDoc1)
+        indexDoc(testIndex, "2", testDoc2)
+
+        executeMonitor(monitor.id)
+
+        val alerts = searchAlertsWithFilter(monitor)
+        assertEquals("Alert saved for test monitor", 4, alerts.size)
+
+        val findings = searchFindings(monitor)
+        assertEquals("Findings saved for test monitor", 4, findings.size)
+
+        val request = """{
+            "size": 0,
+            "query": {
+                "match_all": {}
+            }
+        }"""
+        val httpResponse = adminClient().makeRequest(
+            "GET", "/${monitor.dataSources.queryIndex}/_search",
+            StringEntity(request, ContentType.APPLICATION_JSON)
+        )
+        assertEquals("Search failed", RestStatus.OK, httpResponse.restStatus())
+
+        val searchResponse = SearchResponse.fromXContent(createParser(JsonXContent.jsonXContent, httpResponse.entity.content))
+        searchResponse.hits.totalHits?.let { assertEquals(5L, it.value) }
+    }
+
+    fun `test no of queries generated for document-level monitor based on wildcard indexes`() {
+        val testIndex = createTestIndex("test1")
+        val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(MILLIS))
+        val testDoc = """{
+            "message" : "This is an error from IAD region",
+            "test_strict_date_time" : "$testTime",
+            "test_field" : "us-west-2"
+        }"""
+
+        val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", name = "3", fields = listOf())
+        val docLevelInput = DocLevelMonitorInput("description", listOf("test*"), listOf(docQuery))
+
+        val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN)
+        val monitor = createMonitor(randomDocumentLevelMonitor(inputs = listOf(docLevelInput), triggers = listOf(trigger)))
+        assertNotNull(monitor.id)
+
+        indexDoc(testIndex, "1", testDoc)
+        executeMonitor(monitor.id)
+
+        val request = """{
+            "size": 0,
+            "query": {
+                "match_all": {}
+            }
+        }"""
+        var httpResponse = adminClient().makeRequest(
+            "GET", "/${monitor.dataSources.queryIndex}/_search",
+            StringEntity(request, ContentType.APPLICATION_JSON)
+        )
+        assertEquals("Search failed", RestStatus.OK, httpResponse.restStatus())
+
+        var searchResponse = SearchResponse.fromXContent(createParser(JsonXContent.jsonXContent, httpResponse.entity.content))
+        searchResponse.hits.totalHits?.let { assertEquals(1L, it.value) }
+
+        val testIndex2 = createTestIndex("test2")
+        indexDoc(testIndex2, "1", testDoc)
+        executeMonitor(monitor.id)
+
+        httpResponse = adminClient().makeRequest(
+            "GET", "/${monitor.dataSources.queryIndex}/_search",
+            StringEntity(request, ContentType.APPLICATION_JSON)
+        )
+        assertEquals("Search failed", RestStatus.OK, httpResponse.restStatus())
+
+        searchResponse = SearchResponse.fromXContent(createParser(JsonXContent.jsonXContent, httpResponse.entity.content))
+        searchResponse.hits.totalHits?.let { assertEquals(1L, it.value) }
+    }
+
+    fun `test execute monitor with new index added after first execution that generates alerts and findings from new query`() {
+        val testIndex = createTestIndex("test1")
+        val testIndex2 = createTestIndex("test2")
+        val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(MILLIS))
+        val testDoc = """{
+            "message" : "This is an error from IAD region",
+            "test_strict_date_time" : "$testTime",
+            "test_field" : "us-west-2"
+        }"""
+
+        val docQuery1 = DocLevelQuery(query = "test_field:\"us-west-2\"", name = "3", fields = listOf())
+        val docQuery2 = DocLevelQuery(query = "test_field_new:\"us-west-2\"", name = "4", fields = listOf())
+        val docLevelInput = DocLevelMonitorInput("description", listOf("test*"), listOf(docQuery1, docQuery2))
+
+        val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN)
+        val monitor = createMonitor(randomDocumentLevelMonitor(inputs = listOf(docLevelInput), triggers = listOf(trigger)))
+        assertNotNull(monitor.id)
+
+        indexDoc(testIndex, "1", testDoc)
+        indexDoc(testIndex2, "5", testDoc)
+        executeMonitor(monitor.id)
+
+        var alerts = searchAlertsWithFilter(monitor)
+        assertEquals("Alert saved for test monitor", 2, alerts.size)
+
+        var findings = searchFindings(monitor)
+        assertEquals("Findings saved for test monitor", 2, findings.size)
+
+        var foundFindings = findings.filter { it.relatedDocIds.contains("1") || it.relatedDocIds.contains("5") }
+        assertEquals("Findings saved for test monitor expected 1 and 5", 2, foundFindings.size)
+
+        // clear previous findings and alerts
+        deleteIndex(ALL_FINDING_INDEX_PATTERN)
+        deleteIndex(ALL_ALERT_INDEX_PATTERN)
+
+        val testDocNew = """{
+            "message" : "This is an error from IAD region",
+            "test_strict_date_time" : "$testTime",
+            "test_field_new" : "us-west-2"
+        }"""
+
+        val testIndex3 = createTestIndex("test3")
+        indexDoc(testIndex3, "10", testDocNew)
+
+        val response = executeMonitor(monitor.id)
+
+        val output = entityAsMap(response)
+
+        assertEquals(monitor.name, output["monitor_name"])
+        @Suppress("UNCHECKED_CAST")
+        val searchResult = (output.objectMap("input_results")["results"] as List<Map<String, Any>>).first()
+        @Suppress("UNCHECKED_CAST")
+        val matchingDocsToQuery = searchResult[docQuery2.id] as List<String>
+        assertEquals("Incorrect search result", 1, matchingDocsToQuery.size)
+        assertTrue("Incorrect search result", matchingDocsToQuery.containsAll(listOf("10|$testIndex3")))
+
+        alerts = searchAlertsWithFilter(monitor)
+        assertEquals("Alert saved for test monitor", 1, alerts.size)
+
+        findings = searchFindings(monitor)
+        assertEquals("Findings saved for test monitor", 1, findings.size)
+
+        foundFindings = findings.filter {
+            it.relatedDocIds.contains("10")
+        }
+        assertEquals("Findings saved for test monitor expected 10", 1, foundFindings.size)
+    }
+
+    fun `test document-level monitor when alias only has write index with 0 docs`() {
+        // Monitor should execute, but create 0 findings.
+        val alias = createTestAlias(includeWriteIndex = true)
+        val aliasIndex = alias.keys.first()
+        val query = randomDocLevelQuery(tags = listOf())
+        val input = randomDocLevelMonitorInput(indices = listOf(aliasIndex), queries = listOf(query))
+        val trigger = randomDocumentLevelTrigger(condition = Script("query[id=\"${query.id}\"]"))
+        val monitor = createMonitor(randomDocumentLevelMonitor(enabled = false, inputs = listOf(input), triggers = listOf(trigger)))
+
+        val response: Response
+        try {
+            response = executeMonitor(monitor.id)
+        } catch (e: ResponseException) {
+            assertNotNull("Expected an error message: $e", e.message)
+            e.message?.let {
+                assertTrue("Unexpected exception: $e", it.contains("""reason":"no such index [.opensearch-alerting-findings]"""))
+            }
+            assertEquals(404, e.response.statusLine.statusCode)
+            return
+        }
+
+        val output = entityAsMap(response)
+        val inputResults = output.stringMap("input_results")
+        val errorMessage = inputResults?.get("error")
+        @Suppress("UNCHECKED_CAST")
+        val searchResult = (inputResults?.get("results") as List<Map<String, Any>>).firstOrNull()
+        @Suppress("UNCHECKED_CAST")
+        val findings = searchFindings()
+
+        assertEquals(monitor.name, output["monitor_name"])
+        assertNull("Unexpected monitor execution failure: $errorMessage", errorMessage)
+        findings.findings.forEach {
+            val queryIds = it.finding.docLevelQueries.map { query -> query.id }
+            assertFalse("No findings should exist with queryId ${query.id}, but found: $it", queryIds.contains(query.id))
+        }
+    }
+
+    fun `test document-level monitor when docs exist prior to monitor creation`() {
+        // FIXME: Consider renaming this test case
+        // Only new docs should create findings.
+        val alias = createTestAlias(includeWriteIndex = true)
+        val aliasIndex = alias.keys.first()
+        val indices = alias[aliasIndex]?.keys?.toList() as List<String>
+        val query = randomDocLevelQuery(tags = listOf())
+        val input = randomDocLevelMonitorInput(indices = listOf(aliasIndex), queries = listOf(query))
+        val trigger = randomDocumentLevelTrigger(condition = Script("query[id=\"${query.id}\"]"))
+
+        val preExistingDocIds = mutableSetOf<String>()
+        indices.forEach { index ->
+            val docId = index.hashCode().toString()
+            val doc = """{ "message" : "${query.query}" }"""
+            preExistingDocIds.add(docId)
+            indexDoc(index = index, id = docId, doc = doc)
+        }
+        assertEquals(indices.size, preExistingDocIds.size)
+
+        val monitor = createMonitor(randomDocumentLevelMonitor(enabled = false, inputs = listOf(input), triggers = listOf(trigger)))
+
+        val response = executeMonitor(monitor.id)
+
+        val output = entityAsMap(response)
+        val inputResults = output.stringMap("input_results")
+        val errorMessage = inputResults?.get("error")
+        @Suppress("UNCHECKED_CAST")
+        val searchResult = (inputResults?.get("results") as List<Map<String, Any>>).firstOrNull()
+        @Suppress("UNCHECKED_CAST")
+        val findings = searchFindings()
+
+        assertEquals(monitor.name, output["monitor_name"])
+        assertNull("Unexpected monitor execution failure: $errorMessage", errorMessage)
+        findings.findings.forEach {
+            val docIds = it.finding.relatedDocIds
+            assertTrue(
+                "Findings index should not contain a pre-existing doc, but found $it",
+                preExistingDocIds.intersect(docIds).isEmpty()
+            )
+        }
+    }
+
+    fun `test document-level monitor when alias indices only contain docs that match query`() {
+        // Only new docs should create findings.
+        val alias = createTestAlias(includeWriteIndex = true)
+        val aliasIndex = alias.keys.first()
+        val indices = alias[aliasIndex]?.keys?.toList() as List<String>
+        val query = randomDocLevelQuery(tags = listOf())
+        val input = randomDocLevelMonitorInput(indices = listOf(aliasIndex), queries = listOf(query))
+        val trigger = randomDocumentLevelTrigger(condition = Script("query[id=\"${query.id}\"]"))
+
+        val preExistingDocIds = mutableSetOf<String>()
+        indices.forEach { index ->
+            val docId = index.hashCode().toString()
+            val doc = """{ "message" : "${query.query}" }"""
+            preExistingDocIds.add(docId)
+            indexDoc(index = index, id = docId, doc = doc)
+        }
+        assertEquals(indices.size, preExistingDocIds.size)
+
+        val monitor = createMonitor(randomDocumentLevelMonitor(enabled = false, inputs = listOf(input), triggers = listOf(trigger)))
+        executeMonitor(monitor.id)
+
+        val newDocIds = mutableSetOf<String>()
+        indices.forEach { index ->
+            (1..5).map {
+                val docId = "${index.hashCode()}$it"
+                val doc = """{ "message" : "${query.query}" }"""
+                newDocIds.add(docId)
+                indexDoc(index = index, id = docId, doc = doc)
+            }
+        }
+        assertEquals(indices.size * 5, newDocIds.size)
+
+        val response = executeMonitor(monitor.id)
+
+        val output = entityAsMap(response)
+        val inputResults = output.stringMap("input_results")
+        val errorMessage = inputResults?.get("error")
+        @Suppress("UNCHECKED_CAST")
+        val searchResult = (inputResults?.get("results") as List<Map<String, Any>>).firstOrNull()
+        @Suppress("UNCHECKED_CAST")
+        val findings = searchFindings()
+
+        assertEquals(monitor.name, output["monitor_name"])
+        assertNull("Unexpected monitor execution failure: $errorMessage", errorMessage)
+        findings.findings.forEach {
+            val docIds = it.finding.relatedDocIds
+            assertTrue(
+                "Findings index should not contain a pre-existing doc, but found $it",
+                preExistingDocIds.intersect(docIds).isEmpty()
+            )
+            assertTrue("Found an unexpected finding $it", newDocIds.intersect(docIds).isNotEmpty())
+        }
+    }
+
+    fun `test document-level monitor when alias indices contain docs that do and do not match query`() {
+        // Only matching docs should create findings.
+        val alias = createTestAlias(includeWriteIndex = true)
+        val aliasIndex = alias.keys.first()
+        val indices = alias[aliasIndex]?.keys?.toList() as List<String>
+        val query = randomDocLevelQuery(tags = listOf())
+        val input = randomDocLevelMonitorInput(indices = listOf(aliasIndex), queries = listOf(query))
+        val trigger = randomDocumentLevelTrigger(condition = Script("query[id=\"${query.id}\"]"))
+
+        val preExistingDocIds = mutableSetOf<String>()
+        indices.forEach { index ->
+            val docId = index.hashCode().toString()
+            val doc = """{ "message" : "${query.query}" }"""
+            preExistingDocIds.add(docId)
+            indexDoc(index = index, id = docId, doc = doc)
+        }
+        assertEquals(indices.size, preExistingDocIds.size)
+
+        val monitor = createMonitor(randomDocumentLevelMonitor(enabled = false, inputs = listOf(input), triggers = listOf(trigger)))
+        executeMonitor(monitor.id)
+
+        val matchingDocIds = mutableSetOf<String>()
+        val nonMatchingDocIds = mutableSetOf<String>()
+        indices.forEach { index ->
+            (1..5).map {
+                val matchingDocId = "${index.hashCode()}$it"
+                val matchingDoc = """{ "message" : "${query.query}" }"""
+                indexDoc(index = index, id = matchingDocId, doc = matchingDoc)
+                matchingDocIds.add(matchingDocId)
+
+                val nonMatchingDocId = "${index.hashCode()}${it}2"
+                var nonMatchingDoc = StringBuilder(query.query).insert(2, "difference").toString()
+                nonMatchingDoc = """{ "message" : "$nonMatchingDoc" }"""
+                indexDoc(index = index, id = nonMatchingDocId, doc = nonMatchingDoc)
+                nonMatchingDocIds.add(nonMatchingDocId)
+            }
+        }
+        assertEquals(indices.size * 5, matchingDocIds.size)
+
+        val response = executeMonitor(monitor.id)
+
+        val output = entityAsMap(response)
+        val inputResults = output.stringMap("input_results")
+        val errorMessage = inputResults?.get("error")
+        @Suppress("UNCHECKED_CAST")
+        val searchResult = (inputResults?.get("results") as List<Map<String, Any>>).firstOrNull()
+        @Suppress("UNCHECKED_CAST")
+        val findings = searchFindings()
+
+        assertEquals(monitor.name, output["monitor_name"])
+        assertNull("Unexpected monitor execution failure: $errorMessage", errorMessage)
+        findings.findings.forEach {
+            val docIds = it.finding.relatedDocIds
+            assertTrue(
+                "Findings index should not contain a pre-existing doc, but found $it",
+                preExistingDocIds.intersect(docIds).isEmpty()
+            )
+            assertTrue("Found doc that doesn't match query: $it", nonMatchingDocIds.intersect(docIds).isEmpty())
+            assertFalse("Found an unexpected finding $it", matchingDocIds.intersect(docIds).isNotEmpty())
+        }
+    }
+
+    fun `test document-level monitor when datastreams contain docs that do match query`() {
+        val dataStreamName = "test-datastream"
+        createDataStream(
+            dataStreamName,
+            """
+                "properties" : {
+                  "test_strict_date_time" : { "type" : "date", "format" : "strict_date_time" },
+                  "test_field" : { "type" : "keyword" },
+                  "number" : { "type" : "keyword" }
+                }
+            """.trimIndent(),
+            false
+        )
+
+        val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", name = "3", fields = listOf())
+        val docLevelInput = DocLevelMonitorInput("description", listOf(dataStreamName), listOf(docQuery))
+
+        val action = randomAction(template = randomTemplateScript("Hello {{ctx.monitor.name}}"), destinationId = createDestination().id)
+        val monitor = createMonitor(
+            randomDocumentLevelMonitor(
+                inputs = listOf(docLevelInput),
+                triggers = listOf(randomDocumentLevelTrigger(condition = ALWAYS_RUN, actions = listOf(action)))
+            )
+        )
+
+        val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(MILLIS))
+        val testDoc = """{
+            "@timestamp": "$testTime",
+            "message" : "This is an error from IAD region",
+            "test_strict_date_time" : "$testTime",
+            "test_field" : "us-west-2"
+        }"""
+        indexDoc(dataStreamName, "1", testDoc)
+        var response = executeMonitor(monitor.id)
+        var output = entityAsMap(response)
+        var searchResult = (output.objectMap("input_results")["results"] as List<Map<String, Any>>).first()
+        @Suppress("UNCHECKED_CAST")
+        var matchingDocsToQuery = searchResult[docQuery.id] as List<String>
+        assertEquals("Incorrect search result", 1, matchingDocsToQuery.size)
+
+        rolloverDatastream(dataStreamName)
+        indexDoc(dataStreamName, "2", testDoc)
+        response = executeMonitor(monitor.id)
+        output = entityAsMap(response)
+        searchResult = (output.objectMap("input_results")["results"] as List<Map<String, Any>>).first()
+        @Suppress("UNCHECKED_CAST")
+        matchingDocsToQuery = searchResult[docQuery.id] as List<String>
+        assertEquals("Incorrect search result", 1, matchingDocsToQuery.size)
+
+        deleteDataStream(dataStreamName)
+    }
+
+    fun `test document-level monitor when datastreams contain docs across read-only indices that do match query`() {
+        val dataStreamName = "test-datastream"
+        createDataStream(
+            dataStreamName,
+            """
+                "properties" : {
+                  "test_strict_date_time" : { "type" : "date", "format" : "strict_date_time" },
+                  "test_field" : { "type" : "keyword" },
+                  "number" : { "type" : "keyword" }
+                }
+            """.trimIndent(),
+            false
+        )
+
+        val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", name = "3", fields = listOf())
+        val docLevelInput = DocLevelMonitorInput("description", listOf(dataStreamName), listOf(docQuery))
+
+        val action = randomAction(template = randomTemplateScript("Hello {{ctx.monitor.name}}"), destinationId = createDestination().id)
+        val monitor = createMonitor(
+            randomDocumentLevelMonitor(
+                inputs = listOf(docLevelInput),
+                triggers = listOf(randomDocumentLevelTrigger(condition = ALWAYS_RUN, actions = listOf(action)))
+            )
+        )
+
+        val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(MILLIS))
+        val testDoc = """{
+            "@timestamp": "$testTime",
+            "message" : "This is an error from IAD region",
+            "test_strict_date_time" : "$testTime",
+            "test_field" : "us-west-2"
+        }"""
+        indexDoc(dataStreamName, "1", testDoc)
+        var response = executeMonitor(monitor.id)
+        var output = entityAsMap(response)
+        var searchResult = (output.objectMap("input_results")["results"] as List<Map<String, Any>>).first()
+        @Suppress("UNCHECKED_CAST")
+        var matchingDocsToQuery = searchResult[docQuery.id] as List<String>
+        assertEquals("Incorrect search result", 1, matchingDocsToQuery.size)
+
+        indexDoc(dataStreamName, "2", testDoc)
+        rolloverDatastream(dataStreamName)
+        rolloverDatastream(dataStreamName)
+        indexDoc(dataStreamName, "4", testDoc)
+        rolloverDatastream(dataStreamName)
+        response = executeMonitor(monitor.id)
+        output = entityAsMap(response)
+        searchResult = (output.objectMap("input_results")["results"] as List<Map<String, Any>>).first()
+        @Suppress("UNCHECKED_CAST")
+        matchingDocsToQuery = searchResult[docQuery.id] as List<String>
+        assertEquals("Incorrect search result", 2, matchingDocsToQuery.size)
+
+        indexDoc(dataStreamName, "5", testDoc)
+        indexDoc(dataStreamName, "6", testDoc)
+        response = executeMonitor(monitor.id)
+        output = entityAsMap(response)
+        searchResult = (output.objectMap("input_results")["results"] as List<Map<String, Any>>).first()
+        @Suppress("UNCHECKED_CAST")
+        matchingDocsToQuery = searchResult[docQuery.id] as List<String>
+        assertEquals("Incorrect search result", 2, matchingDocsToQuery.size)
+        deleteDataStream(dataStreamName)
+    }
+
+    fun `test document-level monitor when index alias contain docs that do match query`() {
+        val aliasName = "test-alias"
+        createIndexAlias(
+            aliasName,
+            """
+                "properties" : {
+                  "test_strict_date_time" : { "type" : "date", "format" : "strict_date_time" },
+                  "test_field" : { "type" : "keyword" },
+                  "number" : { "type" : "keyword" }
+                }
+            """.trimIndent()
+        )
+
+        val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", name = "3", fields = listOf())
+        val docLevelInput = DocLevelMonitorInput("description", listOf("$aliasName"), listOf(docQuery))
+
+        val action = randomAction(template = randomTemplateScript("Hello {{ctx.monitor.name}}"), destinationId = createDestination().id)
+        val monitor = createMonitor(
+            randomDocumentLevelMonitor(
+                inputs = listOf(docLevelInput),
+                triggers = listOf(randomDocumentLevelTrigger(condition = ALWAYS_RUN, actions = listOf(action)))
+            )
+        )
+
+        val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(MILLIS))
+        val testDoc = """{
+            "@timestamp": "$testTime",
+            "message" : "This is an error from IAD region",
+            "test_strict_date_time" : "$testTime",
+            "test_field" : "us-west-2"
+        }"""
+        indexDoc(aliasName, "1", testDoc)
+        var response = executeMonitor(monitor.id)
+        var output = entityAsMap(response)
+        var searchResult = (output.objectMap("input_results")["results"] as List<Map<String, Any>>).first()
+        @Suppress("UNCHECKED_CAST")
+        var matchingDocsToQuery = searchResult[docQuery.id] as List<String>
+        assertEquals("Incorrect search result", 1, matchingDocsToQuery.size)
+
+        rolloverDatastream(aliasName)
+        indexDoc(aliasName, "2", testDoc)
+        response = executeMonitor(monitor.id)
+        output = entityAsMap(response)
+        searchResult = (output.objectMap("input_results")["results"] as List<Map<String, Any>>).first()
+        @Suppress("UNCHECKED_CAST")
+        matchingDocsToQuery = searchResult[docQuery.id] as List<String>
+        assertEquals("Incorrect search result", 1, matchingDocsToQuery.size)
+
+        deleteIndexAlias(aliasName)
+    }
+
+    fun `test document-level monitor  when multiple datastreams contain docs across read-only indices that do match query`() {
+        val dataStreamName1 = "test-datastream1"
+        createDataStream(
+            dataStreamName1,
+            """
+                "properties" : {
+                  "test_strict_date_time" : { "type" : "date", "format" : "strict_date_time" },
+                  "test_field" : { "type" : "keyword" },
+                  "number" : { "type" : "keyword" }
+                }
+            """.trimIndent(),
+            false
+        )
+        val dataStreamName2 = "test-datastream2"
+        createDataStream(
+            dataStreamName2,
+            """
+                "properties" : {
+                  "test_strict_date_time" : { "type" : "date", "format" : "strict_date_time" },
+                  "test_field" : { "type" : "keyword" },
+                  "number" : { "type" : "keyword" }
+                }
+            """.trimIndent(),
+            false
+        )
+
+        val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(MILLIS))
+        val testDoc = """{
+            "@timestamp": "$testTime",
+            "message" : "This is an error from IAD region",
+            "test_strict_date_time" : "$testTime",
+            "test_field" : "us-west-2"
+        }"""
+        indexDoc(dataStreamName2, "-1", testDoc)
+        rolloverDatastream(dataStreamName2)
+        indexDoc(dataStreamName2, "0", testDoc)
+
+        val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", name = "3", fields = listOf())
+        val docLevelInput = DocLevelMonitorInput("description", listOf("test-datastream*"), listOf(docQuery))
+
+        val action = randomAction(template = randomTemplateScript("Hello {{ctx.monitor.name}}"), destinationId = createDestination().id)
+        val monitor = createMonitor(
+            randomDocumentLevelMonitor(
+                inputs = listOf(docLevelInput),
+                triggers = listOf(randomDocumentLevelTrigger(condition = ALWAYS_RUN, actions = listOf(action)))
+            )
+        )
+
+        indexDoc(dataStreamName1, "1", testDoc)
+        indexDoc(dataStreamName2, "1", testDoc)
+        var response = executeMonitor(monitor.id)
+        var output = entityAsMap(response)
+        var searchResult = (output.objectMap("input_results")["results"] as List<Map<String, Any>>).first()
+        @Suppress("UNCHECKED_CAST")
+        var matchingDocsToQuery = searchResult[docQuery.id] as List<String>
+        assertEquals("Incorrect search result", 2, matchingDocsToQuery.size)
+
+        indexDoc(dataStreamName1, "2", testDoc)
+        indexDoc(dataStreamName2, "2", testDoc)
+        rolloverDatastream(dataStreamName1)
+        rolloverDatastream(dataStreamName1)
+        rolloverDatastream(dataStreamName2)
+        indexDoc(dataStreamName1, "4", testDoc)
+        indexDoc(dataStreamName2, "4", testDoc)
+        rolloverDatastream(dataStreamName1)
+        response = executeMonitor(monitor.id)
+        output = entityAsMap(response)
+        searchResult = (output.objectMap("input_results")["results"] as List<Map<String, Any>>).first()
+        @Suppress("UNCHECKED_CAST")
+        matchingDocsToQuery = searchResult[docQuery.id] as List<String>
+        assertEquals("Incorrect search result", 4, matchingDocsToQuery.size)
+
+        indexDoc(dataStreamName1, "5", testDoc)
+        indexDoc(dataStreamName1, "6", testDoc)
+        indexDoc(dataStreamName2, "5", testDoc)
+        indexDoc(dataStreamName2, "6", testDoc)
+        response = executeMonitor(monitor.id)
+        output = entityAsMap(response)
+        searchResult = (output.objectMap("input_results")["results"] as List<Map<String, Any>>).first()
+        @Suppress("UNCHECKED_CAST")
+        matchingDocsToQuery = searchResult[docQuery.id] as List<String>
+        assertEquals("Incorrect search result", 4, matchingDocsToQuery.size)
+        deleteDataStream(dataStreamName1)
+        deleteDataStream(dataStreamName2)
+    }
+
+    fun `test document-level monitor ignoring old read-only indices for datastreams`() {
+        val dataStreamName = "test-datastream"
+        createDataStream(
+            dataStreamName,
+            """
+                "properties" : {
+                  "test_strict_date_time" : { "type" : "date", "format" : "strict_date_time" },
+                  "test_field" : { "type" : "keyword" },
+                  "number" : { "type" : "keyword" }
+                }
+            """.trimIndent(),
+            false
+        )
+
+        val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(MILLIS))
+        val testDoc = """{
+            "@timestamp": "$testTime",
+            "message" : "This is an error from IAD region",
+            "test_strict_date_time" : "$testTime",
+            "test_field" : "us-west-2"
+        }"""
+        indexDoc(dataStreamName, "-1", testDoc)
+        rolloverDatastream(dataStreamName)
+        indexDoc(dataStreamName, "0", testDoc)
+
+        val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", name = "3", fields = listOf())
+        val docLevelInput = DocLevelMonitorInput("description", listOf(dataStreamName), listOf(docQuery))
+
+        val action = randomAction(template = randomTemplateScript("Hello {{ctx.monitor.name}}"), destinationId = createDestination().id)
+        val monitor = createMonitor(
+            randomDocumentLevelMonitor(
+                inputs = listOf(docLevelInput),
+                triggers = listOf(randomDocumentLevelTrigger(condition = ALWAYS_RUN, actions = listOf(action)))
+            )
+        )
+
+        indexDoc(dataStreamName, "1", testDoc)
+        var response = executeMonitor(monitor.id)
+        var output = entityAsMap(response)
+        var searchResult = (output.objectMap("input_results")["results"] as List<Map<String, Any>>).first()
+        @Suppress("UNCHECKED_CAST")
+        var matchingDocsToQuery = searchResult[docQuery.id] as List<String>
+        assertEquals("Incorrect search result", 1, matchingDocsToQuery.size)
+
+        rolloverDatastream(dataStreamName)
+        indexDoc(dataStreamName, "2", testDoc)
+        response = executeMonitor(monitor.id)
+        output = entityAsMap(response)
+        searchResult = (output.objectMap("input_results")["results"] as List<Map<String, Any>>).first()
+        @Suppress("UNCHECKED_CAST")
+        matchingDocsToQuery = searchResult[docQuery.id] as List<String>
+        assertEquals("Incorrect search result", 1, matchingDocsToQuery.size)
+
+        deleteDataStream(dataStreamName)
+    }
+
+    fun `test execute monitor with non-null data sources`() {
+
+        val testIndex = createTestIndex()
+        val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(MILLIS))
+        val testDoc = """{
+                "message" : "This is an error from IAD region",
+                "test_strict_date_time" : "$testTime",
+                "test_field" : "us-west-2"
+            }"""
+
+        val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", name = "3", fields = listOf())
+        val docLevelInput = DocLevelMonitorInput("description", listOf(testIndex), listOf(docQuery))
+
+        val alertCategories = AlertCategory.values()
+        val actionExecutionScope = PerAlertActionScope(
+            actionableAlerts = (1..randomInt(alertCategories.size)).map { alertCategories[it - 1] }.toSet()
+        )
+        val actionExecutionPolicy = ActionExecutionPolicy(actionExecutionScope)
+        val actions = (0..randomInt(10)).map {
+            randomActionWithPolicy(
+                template = randomTemplateScript("Hello {{ctx.monitor.name}}"),
+                destinationId = createDestination().id,
+                actionExecutionPolicy = actionExecutionPolicy
+            )
+        }
+
+        val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN, actions = actions)
+        try {
+            createMonitor(
+                randomDocumentLevelMonitor(
+                    inputs = listOf(docLevelInput),
+                    triggers = listOf(trigger),
+                    dataSources = DataSources(
+                        findingsIndex = "custom_findings_index",
+                        alertsIndex = "custom_alerts_index",
+                    )
+                )
+            )
+            fail("Expected create monitor to fail")
         } catch (e: ResponseException) {
-            assertNotNull("Expected an error message: $e", e.message)
-            e.message?.let {
-                assertTrue("Unexpected exception: $e", it.contains("""reason":"no such index [.opensearch-alerting-findings]"""))
-            }
-            assertEquals(404, e.response.statusLine.statusCode)
-            return
+            assertTrue(e.message!!.contains("illegal_argument_exception"))
         }
+    }
+
+    fun `test execute monitor with indices removed after first run`() {
+        val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(MILLIS))
+        val testDoc = """{
+            "message" : "This is an error from IAD region",
+            "test_strict_date_time" : "$testTime",
+            "test_field" : "us-west-2"
+        }"""
+
+        val index1 = createTestIndex()
+        val index2 = createTestIndex()
+        val index4 = createTestIndex()
+        val index5 = createTestIndex()
 
-        val output = entityAsMap(response)
-        val inputResults = output.stringMap("input_results")
-        val errorMessage = inputResults?.get("error")
-        @Suppress("UNCHECKED_CAST")
-        val searchResult = (inputResults?.get("results") as List<Map<String, Any>>).firstOrNull()
-        @Suppress("UNCHECKED_CAST")
-        val findings = searchFindings()
+        val docQuery = DocLevelQuery(query = "\"us-west-2\"", name = "3", fields = listOf())
+        var docLevelInput = DocLevelMonitorInput("description", listOf(index1, index2, index4, index5), listOf(docQuery))
+
+        val action = randomAction(template = randomTemplateScript("Hello {{ctx.monitor.name}}"), destinationId = createDestination().id)
+        val monitor = createMonitor(
+            randomDocumentLevelMonitor(
+                inputs = listOf(docLevelInput),
+                triggers = listOf(randomDocumentLevelTrigger(condition = ALWAYS_RUN, actions = listOf(action)))
+            )
+        )
+
+        indexDoc(index1, "1", testDoc)
+        indexDoc(index2, "1", testDoc)
+        indexDoc(index4, "1", testDoc)
+        indexDoc(index5, "1", testDoc)
+
+        var response = executeMonitor(monitor.id)
 
+        var output = entityAsMap(response)
         assertEquals(monitor.name, output["monitor_name"])
-        assertNull("Unexpected monitor execution failure: $errorMessage", errorMessage)
-        findings.findings.forEach {
-            val queryIds = it.finding.docLevelQueries.map { query -> query.id }
-            assertFalse("No findings should exist with queryId ${query.id}, but found: $it", queryIds.contains(query.id))
-        }
+
+        assertEquals(1, output.objectMap("trigger_results").values.size)
+        deleteIndex(index1)
+        deleteIndex(index2)
+
+        indexDoc(index4, "2", testDoc)
+        response = executeMonitor(monitor.id)
+
+        output = entityAsMap(response)
+        assertEquals(1, output.objectMap("trigger_results").values.size)
     }
 
-    fun `test document-level monitor when docs exist prior to monitor creation`() {
-        // FIXME: Consider renaming this test case
-        // Only new docs should create findings.
-        val alias = createTestAlias(includeWriteIndex = true)
-        val aliasIndex = alias.keys.first()
-        val indices = alias[aliasIndex]?.keys?.toList() as List<String>
-        val query = randomDocLevelQuery(tags = listOf())
-        val input = randomDocLevelMonitorInput(indices = listOf(aliasIndex), queries = listOf(query))
-        val trigger = randomDocumentLevelTrigger(condition = Script("query[id=\"${query.id}\"]"))
+    fun `test execute monitor generates alerts and findings with NOT EQUALS query and EXISTS query`() {
+        val testIndex = createTestIndex()
+        val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(MILLIS))
+        val testDoc = """{
+            "message" : "This is an error from IAD region",
+            "test_strict_date_time" : "$testTime",
+            "test_field" : "us-west-2"
+        }"""
 
-        val preExistingDocIds = mutableSetOf<String>()
-        indices.forEach { index ->
-            val docId = index.hashCode().toString()
-            val doc = """{ "message" : "${query.query}" }"""
-            preExistingDocIds.add(docId)
-            indexDoc(index = index, id = docId, doc = doc)
-        }
-        assertEquals(indices.size, preExistingDocIds.size)
+        val query = "NOT test_field: \"us-east-1\" AND _exists_: test_field"
+        val docQuery = DocLevelQuery(query = query, name = "3", fields = listOf())
+        val docLevelInput = DocLevelMonitorInput("description", listOf(testIndex), listOf(docQuery))
 
-        val monitor = createMonitor(randomDocumentLevelMonitor(enabled = false, inputs = listOf(input), triggers = listOf(trigger)))
+        val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN)
+        val monitor = createMonitor(randomDocumentLevelMonitor(inputs = listOf(docLevelInput), triggers = listOf(trigger)))
+        assertNotNull(monitor.id)
+
+        indexDoc(testIndex, "1", testDoc)
+        indexDoc(testIndex, "5", testDoc)
 
         val response = executeMonitor(monitor.id)
 
         val output = entityAsMap(response)
-        val inputResults = output.stringMap("input_results")
-        val errorMessage = inputResults?.get("error")
+
+        assertEquals(monitor.name, output["monitor_name"])
         @Suppress("UNCHECKED_CAST")
-        val searchResult = (inputResults?.get("results") as List<Map<String, Any>>).firstOrNull()
+        val searchResult = (output.objectMap("input_results")["results"] as List<Map<String, Any>>).first()
         @Suppress("UNCHECKED_CAST")
-        val findings = searchFindings()
+        val matchingDocsToQuery = searchResult[docQuery.id] as List<String>
+        assertEquals("Incorrect search result", 2, matchingDocsToQuery.size)
+        assertTrue("Incorrect search result", matchingDocsToQuery.containsAll(listOf("1|$testIndex", "5|$testIndex")))
 
-        assertEquals(monitor.name, output["monitor_name"])
-        assertNull("Unexpected monitor execution failure: $errorMessage", errorMessage)
-        findings.findings.forEach {
-            val docIds = it.finding.relatedDocIds
-            assertTrue(
-                "Findings index should not contain a pre-existing doc, but found $it",
-                preExistingDocIds.intersect(docIds).isEmpty()
+        val alerts = searchAlertsWithFilter(monitor)
+        assertEquals("Alert saved for test monitor", 2, alerts.size)
+
+        val findings = searchFindings(monitor)
+        assertEquals("Findings saved for test monitor", 2, findings.size)
+        assertTrue("Findings saved for test monitor", findings[0].relatedDocIds.contains("1") || findings[0].relatedDocIds.contains("5"))
+        assertTrue("Findings saved for test monitor", findings[1].relatedDocIds.contains("1") || findings[0].relatedDocIds.contains("5"))
+    }
+
+    fun `test document-level monitor when index alias contain docs that do match a NOT EQUALS query and EXISTS query`() {
+        val aliasName = "test-alias"
+        createIndexAlias(
+            aliasName,
+            """
+                "properties" : {
+                  "test_strict_date_time" : { "type" : "date", "format" : "strict_date_time" },
+                  "test_field" : { "type" : "keyword" },
+                  "number" : { "type" : "keyword" }
+                }
+            """.trimIndent()
+        )
+
+        val docQuery = DocLevelQuery(query = "NOT test_field:\"us-east-1\" AND _exists_: test_field", name = "3", fields = listOf())
+        val docLevelInput = DocLevelMonitorInput("description", listOf("$aliasName"), listOf(docQuery))
+
+        val action = randomAction(template = randomTemplateScript("Hello {{ctx.monitor.name}}"), destinationId = createDestination().id)
+        val monitor = createMonitor(
+            randomDocumentLevelMonitor(
+                inputs = listOf(docLevelInput),
+                triggers = listOf(randomDocumentLevelTrigger(condition = ALWAYS_RUN, actions = listOf(action)))
             )
-        }
+        )
+
+        val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(MILLIS))
+        val testDoc = """{
+            "@timestamp": "$testTime",
+            "message" : "This is an error from IAD region",
+            "test_strict_date_time" : "$testTime",
+            "test_field" : "us-west-2"
+        }"""
+        indexDoc(aliasName, "1", testDoc)
+        var response = executeMonitor(monitor.id)
+        var output = entityAsMap(response)
+        var searchResult = (output.objectMap("input_results")["results"] as List<Map<String, Any>>).first()
+        @Suppress("UNCHECKED_CAST")
+        var matchingDocsToQuery = searchResult[docQuery.id] as List<String>
+        assertEquals("Incorrect search result", 1, matchingDocsToQuery.size)
+
+        rolloverDatastream(aliasName)
+        indexDoc(aliasName, "2", testDoc)
+        response = executeMonitor(monitor.id)
+        output = entityAsMap(response)
+        searchResult = (output.objectMap("input_results")["results"] as List<Map<String, Any>>).first()
+        @Suppress("UNCHECKED_CAST")
+        matchingDocsToQuery = searchResult[docQuery.id] as List<String>
+        assertEquals("Incorrect search result", 1, matchingDocsToQuery.size)
+
+        deleteIndexAlias(aliasName)
     }
 
-    fun `test document-level monitor when alias indices only contain docs that match query`() {
-        // Only new docs should create findings.
-        val alias = createTestAlias(includeWriteIndex = true)
-        val aliasIndex = alias.keys.first()
-        val indices = alias[aliasIndex]?.keys?.toList() as List<String>
-        val query = randomDocLevelQuery(tags = listOf())
-        val input = randomDocLevelMonitorInput(indices = listOf(aliasIndex), queries = listOf(query))
-        val trigger = randomDocumentLevelTrigger(condition = Script("query[id=\"${query.id}\"]"))
+    fun `test execute monitor with wildcard index that generates alerts and findings for NOT EQUALS and EXISTS query operator`() {
+        val testIndexPrefix = "test-index-${randomAlphaOfLength(10).lowercase(Locale.ROOT)}"
+        val testQueryName = "wildcard-test-query"
+        val testIndex = createTestIndex("${testIndexPrefix}1")
+        val testIndex2 = createTestIndex("${testIndexPrefix}2")
 
-        val preExistingDocIds = mutableSetOf<String>()
-        indices.forEach { index ->
-            val docId = index.hashCode().toString()
-            val doc = """{ "message" : "${query.query}" }"""
-            preExistingDocIds.add(docId)
-            indexDoc(index = index, id = docId, doc = doc)
-        }
-        assertEquals(indices.size, preExistingDocIds.size)
+        val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(MILLIS))
+        val testDoc = """{
+            "message" : "This is an error from IAD region",
+            "test_strict_date_time" : "$testTime",
+            "test_field" : "us-west-2"
+        }"""
 
-        val monitor = createMonitor(randomDocumentLevelMonitor(enabled = false, inputs = listOf(input), triggers = listOf(trigger)))
-        executeMonitor(monitor.id)
+        val query = "NOT test_field:\"us-west-1\" AND _exists_: test_field"
+        val docQuery = DocLevelQuery(query = query, name = testQueryName, fields = listOf())
+        val docLevelInput = DocLevelMonitorInput("description", listOf("$testIndexPrefix*"), listOf(docQuery))
 
-        val newDocIds = mutableSetOf<String>()
-        indices.forEach { index ->
-            (1..5).map {
-                val docId = "${index.hashCode()}$it"
-                val doc = """{ "message" : "${query.query}" }"""
-                newDocIds.add(docId)
-                indexDoc(index = index, id = docId, doc = doc)
-            }
-        }
-        assertEquals(indices.size * 5, newDocIds.size)
+        val trigger = randomDocumentLevelTrigger(condition = Script("query[name=$testQueryName]"))
+        val monitor = createMonitor(randomDocumentLevelMonitor(inputs = listOf(docLevelInput), triggers = listOf(trigger)))
+        assertNotNull(monitor.id)
+
+        indexDoc(testIndex, "1", testDoc)
+        indexDoc(testIndex2, "5", testDoc)
+
+        val response = executeMonitor(monitor.id)
+
+        val output = entityAsMap(response)
+
+        assertEquals(monitor.name, output["monitor_name"])
+        @Suppress("UNCHECKED_CAST")
+        val searchResult = (output.objectMap("input_results")["results"] as List<Map<String, Any>>).first()
+        @Suppress("UNCHECKED_CAST")
+        val matchingDocsToQuery = searchResult[docQuery.id] as List<String>
+        assertEquals("Incorrect search result", 2, matchingDocsToQuery.size)
+        assertTrue("Incorrect search result", matchingDocsToQuery.containsAll(listOf("1|$testIndex", "5|$testIndex2")))
+
+        val alerts = searchAlertsWithFilter(monitor)
+        assertEquals("Alert saved for test monitor", 2, alerts.size)
+
+        val findings = searchFindings(monitor)
+        assertEquals("Findings saved for test monitor", 2, findings.size)
+        val foundFindings = findings.filter { it.relatedDocIds.contains("1") || it.relatedDocIds.contains("5") }
+        assertEquals("Didn't find findings for docs 1 and 5", 2, foundFindings.size)
+    }
+
+    fun `test execute monitor with indices having fields with same name different field mappings in multiple indices with NOT EQUALS`() {
+        val testIndex = createTestIndex(
+            "test1",
+            """"properties": {
+                    "source": {
+                        "properties": {
+                            "device": {
+                                "properties": {
+                                    "hwd": {
+                                        "properties": {
+                                            "id": {
+                                                "type":"text",
+                                                "analyzer":"whitespace" 
+                                            }
+                                        }
+                                    } 
+                                }
+                            }
+                        }
+                    },
+                   "test_field" : {
+                        "type":"text" 
+                    }
+                }
+            """.trimIndent()
+        )
+
+        val testIndex2 = createTestIndex(
+            "test2",
+            """"properties": {
+                    "test_field" : {
+                        "type":"keyword"
+                    }
+                }
+            """.trimIndent()
+        )
+
+        val testIndex4 = createTestIndex(
+            "test4",
+            """"properties": {
+                   "source": {
+                        "properties": {
+                            "device": {
+                                "properties": {
+                                    "hwd": {
+                                        "properties": {
+                                            "id": {
+                                                "type":"text"
+                                            }
+                                        }
+                                    } 
+                                }
+                            }
+                        }
+                    },
+                   "test_field" : {
+                        "type":"text" 
+                    }
+                }
+            """.trimIndent()
+        )
 
-        val response = executeMonitor(monitor.id)
+        val testDoc1 = """{
+            "source" : {"device" : {"hwd" : {"id" : "123456"}} },
+            "nested_field": { "test1": "some text" }
+        }"""
+        val testDoc2 = """{
+            "nested_field": { "test1": "some text" },
+            "test_field": "123456"
+        }"""
 
-        val output = entityAsMap(response)
-        val inputResults = output.stringMap("input_results")
-        val errorMessage = inputResults?.get("error")
-        @Suppress("UNCHECKED_CAST")
-        val searchResult = (inputResults?.get("results") as List<Map<String, Any>>).firstOrNull()
-        @Suppress("UNCHECKED_CAST")
-        val findings = searchFindings()
+        val docQuery1 = DocLevelQuery(
+            query = "NOT test_field:\"12345\" AND _exists_: test_field",
+            name = "4",
+            fields = listOf()
+        )
+        val docQuery2 = DocLevelQuery(
+            query = "NOT source.device.hwd.id:\"12345\" AND _exists_: source.device.hwd.id",
+            name = "5",
+            fields = listOf()
+        )
 
-        assertEquals(monitor.name, output["monitor_name"])
-        assertNull("Unexpected monitor execution failure: $errorMessage", errorMessage)
-        findings.findings.forEach {
-            val docIds = it.finding.relatedDocIds
-            assertTrue(
-                "Findings index should not contain a pre-existing doc, but found $it",
-                preExistingDocIds.intersect(docIds).isEmpty()
-            )
-            assertTrue("Found an unexpected finding $it", newDocIds.intersect(docIds).isNotEmpty())
-        }
-    }
+        val docLevelInput = DocLevelMonitorInput("description", listOf("test*"), listOf(docQuery1, docQuery2))
 
-    fun `test document-level monitor when alias indices contain docs that do and do not match query`() {
-        // Only matching docs should create findings.
-        val alias = createTestAlias(includeWriteIndex = true)
-        val aliasIndex = alias.keys.first()
-        val indices = alias[aliasIndex]?.keys?.toList() as List<String>
-        val query = randomDocLevelQuery(tags = listOf())
-        val input = randomDocLevelMonitorInput(indices = listOf(aliasIndex), queries = listOf(query))
-        val trigger = randomDocumentLevelTrigger(condition = Script("query[id=\"${query.id}\"]"))
+        val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN)
+        val monitor = createMonitor(randomDocumentLevelMonitor(inputs = listOf(docLevelInput), triggers = listOf(trigger)))
+        assertNotNull(monitor.id)
 
-        val preExistingDocIds = mutableSetOf<String>()
-        indices.forEach { index ->
-            val docId = index.hashCode().toString()
-            val doc = """{ "message" : "${query.query}" }"""
-            preExistingDocIds.add(docId)
-            indexDoc(index = index, id = docId, doc = doc)
-        }
-        assertEquals(indices.size, preExistingDocIds.size)
+        indexDoc(testIndex4, "1", testDoc1)
+        indexDoc(testIndex2, "1", testDoc2)
+        indexDoc(testIndex, "1", testDoc1)
+        indexDoc(testIndex, "2", testDoc2)
 
-        val monitor = createMonitor(randomDocumentLevelMonitor(enabled = false, inputs = listOf(input), triggers = listOf(trigger)))
         executeMonitor(monitor.id)
 
-        val matchingDocIds = mutableSetOf<String>()
-        val nonMatchingDocIds = mutableSetOf<String>()
-        indices.forEach { index ->
-            (1..5).map {
-                val matchingDocId = "${index.hashCode()}$it"
-                val matchingDoc = """{ "message" : "${query.query}" }"""
-                indexDoc(index = index, id = matchingDocId, doc = matchingDoc)
-                matchingDocIds.add(matchingDocId)
-
-                val nonMatchingDocId = "${index.hashCode()}${it}2"
-                var nonMatchingDoc = StringBuilder(query.query).insert(2, "difference").toString()
-                nonMatchingDoc = """{ "message" : "$nonMatchingDoc" }"""
-                indexDoc(index = index, id = nonMatchingDocId, doc = nonMatchingDoc)
-                nonMatchingDocIds.add(nonMatchingDocId)
-            }
-        }
-        assertEquals(indices.size * 5, matchingDocIds.size)
+        val alerts = searchAlertsWithFilter(monitor)
+        assertEquals("Alert saved for test monitor", 4, alerts.size)
 
-        val response = executeMonitor(monitor.id)
+        val findings = searchFindings(monitor)
+        assertEquals("Findings saved for test monitor", 4, findings.size)
 
-        val output = entityAsMap(response)
-        val inputResults = output.stringMap("input_results")
-        val errorMessage = inputResults?.get("error")
-        @Suppress("UNCHECKED_CAST")
-        val searchResult = (inputResults?.get("results") as List<Map<String, Any>>).firstOrNull()
-        @Suppress("UNCHECKED_CAST")
-        val findings = searchFindings()
+        val request = """{
+            "size": 0,
+            "query": {
+                "match_all": {}
+            }
+        }"""
+        val httpResponse = adminClient().makeRequest(
+            "GET", "/${monitor.dataSources.queryIndex}/_search",
+            StringEntity(request, ContentType.APPLICATION_JSON)
+        )
+        assertEquals("Search failed", RestStatus.OK, httpResponse.restStatus())
 
-        assertEquals(monitor.name, output["monitor_name"])
-        assertNull("Unexpected monitor execution failure: $errorMessage", errorMessage)
-        findings.findings.forEach {
-            val docIds = it.finding.relatedDocIds
-            assertTrue(
-                "Findings index should not contain a pre-existing doc, but found $it",
-                preExistingDocIds.intersect(docIds).isEmpty()
-            )
-            assertTrue("Found doc that doesn't match query: $it", nonMatchingDocIds.intersect(docIds).isEmpty())
-            assertFalse("Found an unexpected finding $it", matchingDocIds.intersect(docIds).isNotEmpty())
-        }
+        val searchResponse = SearchResponse.fromXContent(createParser(JsonXContent.jsonXContent, httpResponse.entity.content))
+        searchResponse.hits.totalHits?.let { assertEquals(5L, it.value) }
     }
 
-    fun `test execute monitor with non-null data sources`() {
-
-        val testIndex = createTestIndex()
-        val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(MILLIS))
-        val testDoc = """{
-                "message" : "This is an error from IAD region",
-                "test_strict_date_time" : "$testTime",
-                "test_field" : "us-west-2"
-            }"""
-
-        val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", name = "3", fields = listOf())
-        val docLevelInput = DocLevelMonitorInput("description", listOf(testIndex), listOf(docQuery))
-
-        val alertCategories = AlertCategory.values()
-        val actionExecutionScope = PerAlertActionScope(
-            actionableAlerts = (1..randomInt(alertCategories.size)).map { alertCategories[it - 1] }.toSet()
+    fun `test execute monitor with indices having fields with same name but different field mappings with NOT EQUALS`() {
+        val testIndex = createTestIndex(
+            "test1",
+            """"properties": {
+                    "source": {
+                        "properties": {
+                            "id": {
+                                "type":"text",
+                                "analyzer":"whitespace" 
+                            }
+                        }
+                    },
+                   "test_field" : {
+                        "type":"text",
+                        "analyzer":"whitespace"
+                    }
+                }
+            """.trimIndent()
         )
-        val actionExecutionPolicy = ActionExecutionPolicy(actionExecutionScope)
-        val actions = (0..randomInt(10)).map {
-            randomActionWithPolicy(
-                template = randomTemplateScript("Hello {{ctx.monitor.name}}"),
-                destinationId = createDestination().id,
-                actionExecutionPolicy = actionExecutionPolicy
-            )
-        }
-
-        val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN, actions = actions)
-        try {
-            createMonitor(
-                randomDocumentLevelMonitor(
-                    inputs = listOf(docLevelInput),
-                    triggers = listOf(trigger),
-                    dataSources = DataSources(
-                        findingsIndex = "custom_findings_index",
-                        alertsIndex = "custom_alerts_index",
-                    )
-                )
-            )
-            fail("Expected create monitor to fail")
-        } catch (e: ResponseException) {
-            assertTrue(e.message!!.contains("illegal_argument_exception"))
-        }
-    }
 
-    fun `test execute monitor with indices removed after first run`() {
-        val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(MILLIS))
+        val testIndex2 = createTestIndex(
+            "test2",
+            """"properties": {
+                    "source": {
+                        "properties": {
+                            "id": {
+                                "type":"text"
+                            }
+                        }
+                    },
+                   "test_field" : {
+                        "type":"text"
+                    }
+                }
+            """.trimIndent()
+        )
         val testDoc = """{
-            "message" : "This is an error from IAD region",
-            "test_strict_date_time" : "$testTime",
-            "test_field" : "us-west-2"
+            "source" : {"id" : "12345" },
+            "nested_field": { "test1": "some text" },
+            "test_field": "12345"
         }"""
 
-        val index1 = createTestIndex()
-        val index2 = createTestIndex()
-        val index4 = createTestIndex()
-        val index5 = createTestIndex()
-
-        val docQuery = DocLevelQuery(query = "\"us-west-2\"", name = "3", fields = listOf())
-        var docLevelInput = DocLevelMonitorInput("description", listOf(index1, index2, index4, index5), listOf(docQuery))
-
-        val action = randomAction(template = randomTemplateScript("Hello {{ctx.monitor.name}}"), destinationId = createDestination().id)
-        val monitor = createMonitor(
-            randomDocumentLevelMonitor(
-                inputs = listOf(docLevelInput),
-                triggers = listOf(randomDocumentLevelTrigger(condition = ALWAYS_RUN, actions = listOf(action)))
-            )
+        val docQuery = DocLevelQuery(
+            query = "(NOT test_field:\"123456\" AND _exists_:test_field) AND source.id:\"12345\"",
+            name = "5",
+            fields = listOf()
         )
+        val docLevelInput = DocLevelMonitorInput("description", listOf("test*"), listOf(docQuery))
 
-        indexDoc(index1, "1", testDoc)
-        indexDoc(index2, "1", testDoc)
-        indexDoc(index4, "1", testDoc)
-        indexDoc(index5, "1", testDoc)
+        val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN)
+        val monitor = createMonitor(randomDocumentLevelMonitor(inputs = listOf(docLevelInput), triggers = listOf(trigger)))
+        assertNotNull(monitor.id)
 
-        var response = executeMonitor(monitor.id)
+        indexDoc(testIndex, "1", testDoc)
+        indexDoc(testIndex2, "1", testDoc)
 
-        var output = entityAsMap(response)
-        assertEquals(monitor.name, output["monitor_name"])
+        executeMonitor(monitor.id)
 
-        assertEquals(1, output.objectMap("trigger_results").values.size)
-        deleteIndex(index1)
-        deleteIndex(index2)
+        val alerts = searchAlertsWithFilter(monitor)
+        assertEquals("Alert saved for test monitor", 2, alerts.size)
 
-        indexDoc(index4, "1", testDoc)
-        response = executeMonitor(monitor.id)
+        val findings = searchFindings(monitor)
+        assertEquals("Findings saved for test monitor", 2, findings.size)
 
-        output = entityAsMap(response)
-        assertEquals(1, output.objectMap("trigger_results").values.size)
+        // as mappings of source.id & test_field are different so, both of them expands
+        val expectedQueries = listOf(
+            "(NOT test_field_test2_${monitor.id}:\"123456\" AND _exists_:test_field_test2_${monitor.id}) " +
+                "AND source.id_test2_${monitor.id}:\"12345\"",
+            "(NOT test_field_test1_${monitor.id}:\"123456\" AND _exists_:test_field_test1_${monitor.id}) " +
+                "AND source.id_test1_${monitor.id}:\"12345\""
+        )
+
+        val request = """{
+            "size": 10,
+            "query": {
+                "match_all": {}
+            }
+        }"""
+        var httpResponse = adminClient().makeRequest(
+            "GET", "/${monitor.dataSources.queryIndex}/_search",
+            StringEntity(request, ContentType.APPLICATION_JSON)
+        )
+        assertEquals("Search failed", RestStatus.OK, httpResponse.restStatus())
+        var searchResponse = SearchResponse.fromXContent(createParser(JsonXContent.jsonXContent, httpResponse.entity.content))
+        searchResponse.hits.forEach { hit ->
+            val query = ((hit.sourceAsMap["query"] as Map<String, Any>)["query_string"] as Map<String, Any>)["query"]
+            assertTrue(expectedQueries.contains(query))
+        }
     }
 
     @Suppress("UNCHECKED_CAST")
diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/MonitorDataSourcesIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/MonitorDataSourcesIT.kt
index ee0be50ac..963d659c3 100644
--- a/alerting/src/test/kotlin/org/opensearch/alerting/MonitorDataSourcesIT.kt
+++ b/alerting/src/test/kotlin/org/opensearch/alerting/MonitorDataSourcesIT.kt
@@ -24,8 +24,6 @@ import org.opensearch.action.fieldcaps.FieldCapabilitiesRequest
 import org.opensearch.action.index.IndexRequest
 import org.opensearch.action.search.SearchRequest
 import org.opensearch.action.support.WriteRequest
-import org.opensearch.alerting.action.SearchMonitorAction
-import org.opensearch.alerting.action.SearchMonitorRequest
 import org.opensearch.alerting.alerts.AlertIndices
 import org.opensearch.alerting.core.ScheduledJobIndices
 import org.opensearch.alerting.model.DocumentLevelTriggerRunResult
@@ -47,6 +45,7 @@ import org.opensearch.commons.alerting.action.DeleteMonitorRequest
 import org.opensearch.commons.alerting.action.GetAlertsRequest
 import org.opensearch.commons.alerting.action.GetAlertsResponse
 import org.opensearch.commons.alerting.action.IndexMonitorResponse
+import org.opensearch.commons.alerting.action.SearchMonitorRequest
 import org.opensearch.commons.alerting.aggregation.bucketselectorext.BucketSelectorExtAggregationBuilder
 import org.opensearch.commons.alerting.model.Alert
 import org.opensearch.commons.alerting.model.ChainedAlertTrigger
@@ -367,6 +366,204 @@ class MonitorDataSourcesIT : AlertingSingleNodeTestCase() {
         assertEquals("Didn't match query", 1, findings[0].docLevelQueries.size)
     }
 
+    fun `test all fields fetched and submitted to percolate query when one of the queries doesn't have queryFieldNames`() {
+        // doesn't have query field names so even if other queries pass the wrong fields to query, findings will get generated on matching docs
+        val docQuery1 = DocLevelQuery(
+            query = "source.ip.v6.v1:12345",
+            name = "3",
+            fields = listOf()
+        )
+        val docQuery2 = DocLevelQuery(
+            query = "source.ip.v6.v2:16645",
+            name = "4",
+            fields = listOf(),
+            queryFieldNames = listOf("alias.some.fff", "source.ip.v6.v1")
+        )
+        val docQuery3 = DocLevelQuery(
+            query = "source.ip.v4.v0:120",
+            name = "5",
+            fields = listOf(),
+            queryFieldNames = listOf("alias.some.fff", "source.ip.v6.v1")
+        )
+        val docQuery4 =
+            DocLevelQuery(
+                query = "alias.some.fff:\"us-west-2\"",
+                name = "6",
+                fields = listOf(),
+                queryFieldNames = listOf("alias.some.fff", "source.ip.v6.v1")
+            )
+        val docQuery5 = DocLevelQuery(
+            query = "message:\"This is an error from IAD region\"",
+            name = "7",
+            queryFieldNames = listOf("alias.some.fff", "source.ip.v6.v1"),
+            fields = listOf()
+        )
+        val docQuery6 =
+            DocLevelQuery(
+                query = "type.subtype:\"some subtype\"",
+                name = "8",
+                fields = listOf(),
+                queryFieldNames = listOf("alias.some.fff", "source.ip.v6.v1")
+            )
+        val docQuery7 =
+            DocLevelQuery(
+                query = "supertype.type:\"some type\"",
+                name = "9",
+                fields = listOf(),
+                queryFieldNames = listOf("alias.some.fff", "source.ip.v6.v1")
+            )
+        val docLevelInput = DocLevelMonitorInput(
+            "description", listOf(index), listOf(docQuery1, docQuery2, docQuery3, docQuery4, docQuery5, docQuery6, docQuery7)
+        )
+        val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN)
+        val customFindingsIndex = "custom_findings_index"
+        val customFindingsIndexPattern = "custom_findings_index-1"
+        val customQueryIndex = "custom_alerts_index"
+        var monitor = randomDocumentLevelMonitor(
+            inputs = listOf(docLevelInput),
+            triggers = listOf(trigger),
+            dataSources = DataSources(
+                queryIndex = customQueryIndex,
+                findingsIndex = customFindingsIndex,
+                findingsIndexPattern = customFindingsIndexPattern
+            )
+        )
+        val monitorResponse = createMonitor(monitor)
+        val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(MILLIS))
+        // Trying to test here few different "nesting" situations and "wierd" characters
+        val testDoc = """{
+            "message" : "This is an error from IAD region",
+            "source.ip.v6.v1" : 12345,
+            "source.ip.v6.v2" : 16645,
+            "source.ip.v4.v0" : 120,
+            "test_bad_char" : "\u0000", 
+            "test_strict_date_time" : "$testTime",
+            "test_field.some_other_field" : "us-west-2",
+            "type.subtype" : "some subtype",
+            "supertype.type" : "some type"
+        }"""
+        indexDoc(index, "1", testDoc)
+        client().admin().indices().putMapping(
+            PutMappingRequest(index).source("alias.some.fff", "type=alias,path=test_field.some_other_field")
+        )
+        assertFalse(monitorResponse?.id.isNullOrEmpty())
+        monitor = monitorResponse!!.monitor
+        val id = monitorResponse.id
+        val executeMonitorResponse = executeMonitor(monitor, id, false)
+        Assert.assertEquals(executeMonitorResponse!!.monitorRunResult.monitorName, monitor.name)
+        Assert.assertEquals(executeMonitorResponse.monitorRunResult.triggerResults.size, 1)
+        searchAlerts(id)
+        val table = Table("asc", "id", null, 1, 0, "")
+        var getAlertsResponse = client()
+            .execute(AlertingActions.GET_ALERTS_ACTION_TYPE, GetAlertsRequest(table, "ALL", "ALL", null, null))
+            .get()
+        Assert.assertTrue(getAlertsResponse != null)
+        Assert.assertTrue(getAlertsResponse.alerts.size == 1)
+        val findings = searchFindings(id, customFindingsIndex)
+        assertEquals("Findings saved for test monitor", 1, findings.size)
+        assertTrue("Findings saved for test monitor", findings[0].relatedDocIds.contains("1"))
+        assertEquals("Didn't match all 7 queries", 7, findings[0].docLevelQueries.size)
+    }
+
+    fun `test percolate query failure when queryFieldNames has alias`() {
+        // doesn't have query field names so even if other queries pass the wrong fields to query, findings will get generated on matching docs
+        val docQuery1 = DocLevelQuery(
+            query = "source.ip.v6.v1:12345",
+            name = "3",
+            fields = listOf(),
+            queryFieldNames = listOf("alias.some.fff", "source.ip.v6.v1")
+        )
+        val docQuery2 = DocLevelQuery(
+            query = "source.ip.v6.v2:16645",
+            name = "4",
+            fields = listOf(),
+            queryFieldNames = listOf("source.ip.v6.v2")
+        )
+        val docQuery3 = DocLevelQuery(
+            query = "source.ip.v4.v0:120",
+            name = "5",
+            fields = listOf(),
+            queryFieldNames = listOf("source.ip.v6.v4")
+        )
+        val docQuery4 =
+            DocLevelQuery(
+                query = "alias.some.fff:\"us-west-2\"",
+                name = "6",
+                fields = listOf(),
+                queryFieldNames = listOf("alias.some.fff")
+            )
+        val docQuery5 = DocLevelQuery(
+            query = "message:\"This is an error from IAD region\"",
+            name = "7",
+            queryFieldNames = listOf("alias.some.fff", "source.ip.v6.v1"),
+            fields = listOf()
+        )
+        val docQuery6 =
+            DocLevelQuery(
+                query = "type.subtype:\"some subtype\"",
+                name = "8",
+                fields = listOf(),
+                queryFieldNames = listOf("alias.some.fff", "source.ip.v6.v1")
+            )
+        val docQuery7 =
+            DocLevelQuery(
+                query = "supertype.type:\"some type\"",
+                name = "9",
+                fields = listOf(),
+                queryFieldNames = listOf("alias.some.fff", "source.ip.v6.v1")
+            )
+        val docLevelInput = DocLevelMonitorInput(
+            "description", listOf(index), listOf(docQuery1, docQuery2, docQuery3, docQuery4, docQuery5, docQuery6, docQuery7)
+        )
+        val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN)
+        val customFindingsIndex = "custom_findings_index"
+        val customFindingsIndexPattern = "custom_findings_index-1"
+        val customQueryIndex = "custom_alerts_index"
+        var monitor = randomDocumentLevelMonitor(
+            inputs = listOf(docLevelInput),
+            triggers = listOf(trigger),
+            dataSources = DataSources(
+                queryIndex = customQueryIndex,
+                findingsIndex = customFindingsIndex,
+                findingsIndexPattern = customFindingsIndexPattern
+            )
+        )
+        val monitorResponse = createMonitor(monitor)
+        val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(MILLIS))
+        // Trying to test here few different "nesting" situations and "wierd" characters
+        val testDoc = """{
+            "message" : "This is an error from IAD region",
+            "source.ip.v6.v1" : 12345,
+            "source.ip.v6.v2" : 16645,
+            "source.ip.v4.v0" : 120,
+            "test_bad_char" : "\u0000", 
+            "test_strict_date_time" : "$testTime",
+            "test_field.some_other_field" : "us-west-2",
+            "type.subtype" : "some subtype",
+            "supertype.type" : "some type"
+        }"""
+        indexDoc(index, "1", testDoc)
+        client().admin().indices().putMapping(
+            PutMappingRequest(index).source("alias.some.fff", "type=alias,path=test_field.some_other_field")
+        )
+        assertFalse(monitorResponse?.id.isNullOrEmpty())
+        monitor = monitorResponse!!.monitor
+        val id = monitorResponse.id
+        val executeMonitorResponse = executeMonitor(monitor, id, false)
+        Assert.assertEquals(executeMonitorResponse!!.monitorRunResult.monitorName, monitor.name)
+        Assert.assertEquals(executeMonitorResponse.monitorRunResult.triggerResults.size, 0)
+        searchAlerts(id)
+        val table = Table("asc", "id", null, 1, 0, "")
+        var getAlertsResponse = client()
+            .execute(AlertingActions.GET_ALERTS_ACTION_TYPE, GetAlertsRequest(table, "ALL", "ALL", null, null))
+            .get()
+        Assert.assertTrue(getAlertsResponse != null)
+        Assert.assertTrue(getAlertsResponse.alerts.size == 1)
+        Assert.assertTrue(getAlertsResponse.alerts[0].state.toString().equals(Alert.State.ERROR.toString()))
+        val findings = searchFindings(id, customFindingsIndex)
+        assertEquals("Findings saved for test monitor", 0, findings.size)
+    }
+
     fun `test execute monitor with custom query index`() {
         val q1 = DocLevelQuery(query = "source.ip.v6.v1:12345", name = "3", fields = listOf())
         val q2 = DocLevelQuery(query = "source.ip.v6.v2:16645", name = "4", fields = listOf())
@@ -1447,12 +1644,12 @@ class MonitorDataSourcesIT : AlertingSingleNodeTestCase() {
         Assert.assertTrue(getAlertsResponse.alerts.size == 1)
         val searchRequest = SearchRequest(SCHEDULED_JOBS_INDEX)
         var searchMonitorResponse =
-            client().execute(SearchMonitorAction.INSTANCE, SearchMonitorRequest(searchRequest))
+            client().execute(AlertingActions.SEARCH_MONITORS_ACTION_TYPE, SearchMonitorRequest(searchRequest))
                 .get()
         Assert.assertEquals(searchMonitorResponse.hits.hits.size, 0)
         searchRequest.source().query(MatchQueryBuilder("monitor.owner", "security_analytics_plugin"))
         searchMonitorResponse =
-            client().execute(SearchMonitorAction.INSTANCE, SearchMonitorRequest(searchRequest))
+            client().execute(AlertingActions.SEARCH_MONITORS_ACTION_TYPE, SearchMonitorRequest(searchRequest))
                 .get()
         Assert.assertEquals(searchMonitorResponse.hits.hits.size, 1)
     }
diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/MonitorRunnerServiceIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/MonitorRunnerServiceIT.kt
index 72b7c0423..ff79afffb 100644
--- a/alerting/src/test/kotlin/org/opensearch/alerting/MonitorRunnerServiceIT.kt
+++ b/alerting/src/test/kotlin/org/opensearch/alerting/MonitorRunnerServiceIT.kt
@@ -969,7 +969,7 @@ class MonitorRunnerServiceIT : AlertingRestTestCase() {
         // GIVEN
         val indices = (1..5).map { createTestIndex() }.toTypedArray()
         val pathParams = indices.joinToString(",")
-        val path = "/_cluster/health/"
+        val path = "/_cluster/health"
         val input = randomClusterMetricsInput(
             path = path,
             pathParams = pathParams
diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/action/GetMonitorActionTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/action/GetMonitorActionTests.kt
deleted file mode 100644
index 3d1a5dffa..000000000
--- a/alerting/src/test/kotlin/org/opensearch/alerting/action/GetMonitorActionTests.kt
+++ /dev/null
@@ -1,16 +0,0 @@
-/*
- * Copyright OpenSearch Contributors
- * SPDX-License-Identifier: Apache-2.0
- */
-
-package org.opensearch.alerting.action
-
-import org.opensearch.test.OpenSearchTestCase
-
-class GetMonitorActionTests : OpenSearchTestCase() {
-
-    fun `test get monitor action name`() {
-        assertNotNull(GetMonitorAction.INSTANCE.name())
-        assertEquals(GetMonitorAction.INSTANCE.name(), GetMonitorAction.NAME)
-    }
-}
diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/action/GetMonitorRequestTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/action/GetMonitorRequestTests.kt
deleted file mode 100644
index c8d8731c5..000000000
--- a/alerting/src/test/kotlin/org/opensearch/alerting/action/GetMonitorRequestTests.kt
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Copyright OpenSearch Contributors
- * SPDX-License-Identifier: Apache-2.0
- */
-
-package org.opensearch.alerting.action
-
-import org.opensearch.common.io.stream.BytesStreamOutput
-import org.opensearch.core.common.io.stream.StreamInput
-import org.opensearch.rest.RestRequest
-import org.opensearch.search.fetch.subphase.FetchSourceContext
-import org.opensearch.test.OpenSearchTestCase
-
-class GetMonitorRequestTests : OpenSearchTestCase() {
-
-    fun `test get monitor request`() {
-
-        val req = GetMonitorRequest("1234", 1L, RestRequest.Method.GET, FetchSourceContext.FETCH_SOURCE)
-        assertNotNull(req)
-
-        val out = BytesStreamOutput()
-        req.writeTo(out)
-        val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes)
-        val newReq = GetMonitorRequest(sin)
-        assertEquals("1234", newReq.monitorId)
-        assertEquals(1L, newReq.version)
-        assertEquals(RestRequest.Method.GET, newReq.method)
-        assertEquals(FetchSourceContext.FETCH_SOURCE, newReq.srcContext)
-    }
-
-    fun `test get monitor request without src context`() {
-
-        val req = GetMonitorRequest("1234", 1L, RestRequest.Method.GET, null)
-        assertNotNull(req)
-
-        val out = BytesStreamOutput()
-        req.writeTo(out)
-        val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes)
-        val newReq = GetMonitorRequest(sin)
-        assertEquals("1234", newReq.monitorId)
-        assertEquals(1L, newReq.version)
-        assertEquals(RestRequest.Method.GET, newReq.method)
-        assertEquals(null, newReq.srcContext)
-    }
-
-    fun `test head monitor request`() {
-
-        val req = GetMonitorRequest("1234", 2L, RestRequest.Method.HEAD, FetchSourceContext.FETCH_SOURCE)
-        assertNotNull(req)
-
-        val out = BytesStreamOutput()
-        req.writeTo(out)
-        val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes)
-        val newReq = GetMonitorRequest(sin)
-        assertEquals("1234", newReq.monitorId)
-        assertEquals(2L, newReq.version)
-        assertEquals(RestRequest.Method.HEAD, newReq.method)
-        assertEquals(FetchSourceContext.FETCH_SOURCE, newReq.srcContext)
-    }
-}
diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/action/GetMonitorResponseTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/action/GetMonitorResponseTests.kt
deleted file mode 100644
index af7b0e5c9..000000000
--- a/alerting/src/test/kotlin/org/opensearch/alerting/action/GetMonitorResponseTests.kt
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright OpenSearch Contributors
- * SPDX-License-Identifier: Apache-2.0
- */
-
-package org.opensearch.alerting.action
-
-import org.opensearch.alerting.randomUser
-import org.opensearch.common.io.stream.BytesStreamOutput
-import org.opensearch.commons.alerting.model.CronSchedule
-import org.opensearch.commons.alerting.model.Monitor
-import org.opensearch.core.common.io.stream.StreamInput
-import org.opensearch.core.rest.RestStatus
-import org.opensearch.test.OpenSearchTestCase
-import java.time.Instant
-import java.time.ZoneId
-
-class GetMonitorResponseTests : OpenSearchTestCase() {
-
-    fun `test get monitor response`() {
-        val req = GetMonitorResponse("1234", 1L, 2L, 0L, RestStatus.OK, null, null)
-        assertNotNull(req)
-
-        val out = BytesStreamOutput()
-        req.writeTo(out)
-        val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes)
-        val newReq = GetMonitorResponse(sin)
-        assertEquals("1234", newReq.id)
-        assertEquals(1L, newReq.version)
-        assertEquals(RestStatus.OK, newReq.status)
-        assertEquals(null, newReq.monitor)
-    }
-
-    fun `test get monitor response with monitor`() {
-        val cronExpression = "31 * * * *" // Run at minute 31.
-        val testInstance = Instant.ofEpochSecond(1538164858L)
-
-        val cronSchedule = CronSchedule(cronExpression, ZoneId.of("Asia/Kolkata"), testInstance)
-        val monitor = Monitor(
-            id = "123",
-            version = 0L,
-            name = "test-monitor",
-            enabled = true,
-            schedule = cronSchedule,
-            lastUpdateTime = Instant.now(),
-            enabledTime = Instant.now(),
-            monitorType = Monitor.MonitorType.QUERY_LEVEL_MONITOR,
-            user = randomUser(),
-            schemaVersion = 0,
-            inputs = mutableListOf(),
-            triggers = mutableListOf(),
-            uiMetadata = mutableMapOf()
-        )
-        val req = GetMonitorResponse("1234", 1L, 2L, 0L, RestStatus.OK, monitor, null)
-        assertNotNull(req)
-
-        val out = BytesStreamOutput()
-        req.writeTo(out)
-        val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes)
-        val newReq = GetMonitorResponse(sin)
-        assertEquals("1234", newReq.id)
-        assertEquals(1L, newReq.version)
-        assertEquals(RestStatus.OK, newReq.status)
-        assertNotNull(newReq.monitor)
-    }
-}
diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/action/SearchMonitorActionTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/action/SearchMonitorActionTests.kt
deleted file mode 100644
index 61f96529d..000000000
--- a/alerting/src/test/kotlin/org/opensearch/alerting/action/SearchMonitorActionTests.kt
+++ /dev/null
@@ -1,17 +0,0 @@
-/*
- * Copyright OpenSearch Contributors
- * SPDX-License-Identifier: Apache-2.0
- */
-
-package org.opensearch.alerting.action
-
-import org.junit.Assert
-import org.opensearch.test.OpenSearchTestCase
-
-class SearchMonitorActionTests : OpenSearchTestCase() {
-
-    fun `test search monitor action name`() {
-        Assert.assertNotNull(SearchMonitorAction.INSTANCE.name())
-        Assert.assertEquals(SearchMonitorAction.INSTANCE.name(), SearchMonitorAction.NAME)
-    }
-}
diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/action/SearchMonitorRequestTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/action/SearchMonitorRequestTests.kt
deleted file mode 100644
index 43a5bc873..000000000
--- a/alerting/src/test/kotlin/org/opensearch/alerting/action/SearchMonitorRequestTests.kt
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright OpenSearch Contributors
- * SPDX-License-Identifier: Apache-2.0
- */
-
-package org.opensearch.alerting.action
-
-import org.opensearch.action.search.SearchRequest
-import org.opensearch.common.io.stream.BytesStreamOutput
-import org.opensearch.common.unit.TimeValue
-import org.opensearch.core.common.io.stream.StreamInput
-import org.opensearch.search.builder.SearchSourceBuilder
-import org.opensearch.test.OpenSearchTestCase
-import org.opensearch.test.rest.OpenSearchRestTestCase
-import java.util.concurrent.TimeUnit
-
-class SearchMonitorRequestTests : OpenSearchTestCase() {
-
-    fun `test search monitors request`() {
-        val searchSourceBuilder = SearchSourceBuilder().from(0).size(100).timeout(TimeValue(60, TimeUnit.SECONDS))
-        val searchRequest = SearchRequest().indices(OpenSearchRestTestCase.randomAlphaOfLength(10)).source(searchSourceBuilder)
-        val searchMonitorRequest = SearchMonitorRequest(searchRequest)
-        assertNotNull(searchMonitorRequest)
-
-        val out = BytesStreamOutput()
-        searchMonitorRequest.writeTo(out)
-        val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes)
-        val newReq = SearchMonitorRequest(sin)
-
-        assertNotNull(newReq.searchRequest)
-        assertEquals(1, newReq.searchRequest.indices().size)
-    }
-}
diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/model/WriteableTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/model/WriteableTests.kt
index 6851c471d..6ef15f8d8 100644
--- a/alerting/src/test/kotlin/org/opensearch/alerting/model/WriteableTests.kt
+++ b/alerting/src/test/kotlin/org/opensearch/alerting/model/WriteableTests.kt
@@ -40,7 +40,10 @@ class WriteableTests : OpenSearchTestCase() {
         runResult.writeTo(out)
         val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes)
         val newRunResult = QueryLevelTriggerRunResult(sin)
-        assertEquals("Round tripping ActionRunResult doesn't work", runResult, newRunResult)
+        assertEquals(runResult.triggerName, newRunResult.triggerName)
+        assertEquals(runResult.triggered, newRunResult.triggered)
+        assertEquals(runResult.error, newRunResult.error)
+        assertEquals(runResult.actionResults, newRunResult.actionResults)
     }
 
     fun `test bucket-level triggerrunresult as stream`() {
diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/FindingsRestApiIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/FindingsRestApiIT.kt
index 1839bc807..bc9f1261c 100644
--- a/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/FindingsRestApiIT.kt
+++ b/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/FindingsRestApiIT.kt
@@ -31,6 +31,33 @@ class FindingsRestApiIT : AlertingRestTestCase() {
         assertFalse(response.findings[0].documents[0].found)
     }
 
+    fun `test find Finding where source docData is null`() {
+        val testIndex = createTestIndex()
+        val testDoc = """{
+            "message" : "This is an error from IAD region",
+            "test_field" : "us-west-2"
+        }"""
+        indexDoc(testIndex, "someId", testDoc)
+
+        val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", name = "3", fields = listOf())
+        val docLevelInput = DocLevelMonitorInput("description", listOf(testIndex), listOf(docQuery))
+        val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN)
+        val trueMonitor = createMonitor(randomDocumentLevelMonitor(inputs = listOf(docLevelInput), triggers = listOf(trigger)))
+        executeMonitor(trueMonitor.id, mapOf(Pair("dryrun", "true")))
+
+        createFinding(matchingDocIds = listOf("someId"), index = testIndex)
+        val responseBeforeDelete = searchFindings()
+        assertEquals(1, responseBeforeDelete.totalFindings)
+        assertEquals(1, responseBeforeDelete.findings[0].documents.size)
+        assertTrue(responseBeforeDelete.findings[0].documents[0].found)
+
+        deleteDoc(testIndex, "someId")
+        val responseAfterDelete = searchFindings()
+        assertEquals(1, responseAfterDelete.totalFindings)
+        assertEquals(1, responseAfterDelete.findings[0].documents.size)
+        assertFalse(responseAfterDelete.findings[0].documents[0].found)
+    }
+
     fun `test find Finding where doc is retrieved`() {
         val testIndex = createTestIndex()
         val testDoc = """{
diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/SecureWorkflowRestApiIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/SecureWorkflowRestApiIT.kt
index 1c838aaeb..6d0112c52 100644
--- a/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/SecureWorkflowRestApiIT.kt
+++ b/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/SecureWorkflowRestApiIT.kt
@@ -394,6 +394,7 @@ class SecureWorkflowRestApiIT : AlertingRestTestCase() {
             assertEquals("Get workflow failed", RestStatus.FORBIDDEN.status, e.response.statusLine.statusCode)
         } finally {
             deleteRoleAndRoleMapping(TEST_HR_ROLE)
+            deleteRoleMapping(ALERTING_FULL_ACCESS_ROLE)
             deleteUser(getUser)
             getUserClient?.close()
         }
@@ -444,6 +445,7 @@ class SecureWorkflowRestApiIT : AlertingRestTestCase() {
         } catch (e: ResponseException) {
             assertEquals("Create workflow failed", RestStatus.FORBIDDEN.status, e.response.statusLine.statusCode)
         } finally {
+            deleteRoleMapping(ALERTING_FULL_ACCESS_ROLE)
             deleteUser(userWithDifferentRole)
             userWithDifferentRoleClient?.close()
         }
@@ -1101,6 +1103,7 @@ class SecureWorkflowRestApiIT : AlertingRestTestCase() {
             assertEquals("Delete workflow failed", RestStatus.OK, response?.restStatus())
         } finally {
             deleteRoleAndRoleMapping(TEST_HR_ROLE)
+            deleteRoleMapping(ALERTING_FULL_ACCESS_ROLE)
             deleteUser(deleteUser)
             deleteUserClient?.close()
         }
diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/WorkflowRestApiIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/WorkflowRestApiIT.kt
index 8c073c4b6..94c7e940e 100644
--- a/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/WorkflowRestApiIT.kt
+++ b/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/WorkflowRestApiIT.kt
@@ -37,6 +37,8 @@ import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder
 import org.opensearch.search.builder.SearchSourceBuilder
 import org.opensearch.test.junit.annotations.TestLogging
 import java.time.Instant
+import java.time.ZonedDateTime
+import java.time.format.DateTimeFormatter
 import java.time.temporal.ChronoUnit
 import java.util.Collections
 import java.util.Locale
@@ -1185,4 +1187,45 @@ class WorkflowRestApiIT : AlertingRestTestCase() {
         val findings = searchFindings(monitor.copy(id = monitorResponse.id))
         assertEquals("Findings saved for test monitor", 1, findings.size)
     }
+
+    fun `test workflow run generates no error alerts with versionconflictengineexception with locks`() {
+        val testIndex = createTestIndex()
+        val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(ChronoUnit.MILLIS))
+        val testDoc = """{
+            "message" : "This is an error from IAD region",
+            "test_strict_date_time" : "$testTime",
+            "test_field" : "us-west-2"
+        }"""
+
+        val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", name = "3", fields = listOf())
+        val docLevelInput = DocLevelMonitorInput("description", listOf(testIndex), listOf(docQuery))
+
+        val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN)
+        val monitor = createMonitor(
+            randomDocumentLevelMonitor(
+                name = "__lag-monitor-test__",
+                inputs = listOf(docLevelInput),
+                triggers = listOf(trigger),
+                enabled = false,
+                schedule = IntervalSchedule(interval = 1, unit = ChronoUnit.MINUTES)
+            )
+        )
+        assertNotNull(monitor.id)
+        createWorkflow(
+            randomWorkflow(
+                monitorIds = listOf(monitor.id),
+                enabled = true,
+                schedule = IntervalSchedule(1, ChronoUnit.MINUTES)
+            )
+        )
+
+        indexDoc(testIndex, "1", testDoc)
+        indexDoc(testIndex, "5", testDoc)
+        Thread.sleep(240000)
+
+        val alerts = searchAlerts(monitor)
+        alerts.forEach {
+            assertTrue(it.errorMessage == null)
+        }
+    }
 }
diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/transport/AlertingSingleNodeTestCase.kt b/alerting/src/test/kotlin/org/opensearch/alerting/transport/AlertingSingleNodeTestCase.kt
index 46b656b94..f1f8882f7 100644
--- a/alerting/src/test/kotlin/org/opensearch/alerting/transport/AlertingSingleNodeTestCase.kt
+++ b/alerting/src/test/kotlin/org/opensearch/alerting/transport/AlertingSingleNodeTestCase.kt
@@ -22,8 +22,6 @@ import org.opensearch.alerting.action.ExecuteMonitorResponse
 import org.opensearch.alerting.action.ExecuteWorkflowAction
 import org.opensearch.alerting.action.ExecuteWorkflowRequest
 import org.opensearch.alerting.action.ExecuteWorkflowResponse
-import org.opensearch.alerting.action.GetMonitorAction
-import org.opensearch.alerting.action.GetMonitorRequest
 import org.opensearch.alerting.alerts.AlertIndices
 import org.opensearch.alerting.model.MonitorMetadata
 import org.opensearch.alerting.model.WorkflowMetadata
@@ -37,6 +35,7 @@ import org.opensearch.commons.alerting.action.DeleteMonitorRequest
 import org.opensearch.commons.alerting.action.DeleteWorkflowRequest
 import org.opensearch.commons.alerting.action.GetFindingsRequest
 import org.opensearch.commons.alerting.action.GetFindingsResponse
+import org.opensearch.commons.alerting.action.GetMonitorRequest
 import org.opensearch.commons.alerting.action.GetWorkflowAlertsRequest
 import org.opensearch.commons.alerting.action.GetWorkflowAlertsResponse
 import org.opensearch.commons.alerting.action.GetWorkflowRequest
@@ -346,7 +345,7 @@ abstract class AlertingSingleNodeTestCase : OpenSearchSingleNodeTestCase() {
         version: Long = 1L,
         fetchSourceContext: FetchSourceContext = FetchSourceContext.FETCH_SOURCE,
     ) = client().execute(
-        GetMonitorAction.INSTANCE,
+        AlertingActions.GET_MONITOR_ACTION_TYPE,
         GetMonitorRequest(monitorId, version, RestRequest.Method.GET, fetchSourceContext)
     ).get()
 
diff --git a/build-tools/merged-coverage.gradle b/build-tools/merged-coverage.gradle
index c4e07ab75..a5bb24894 100644
--- a/build-tools/merged-coverage.gradle
+++ b/build-tools/merged-coverage.gradle
@@ -5,7 +5,7 @@
 
 allprojects {
     plugins.withId('jacoco') {
-        jacoco.toolVersion = '0.8.7'
+        jacoco.toolVersion = '0.8.11'
         // For some reason this dependency isn't getting setup automatically by the jacoco plugin
         tasks.withType(JacocoReport) { 
             dependsOn tasks.withType(Test)
@@ -13,26 +13,18 @@ allprojects {
     }
 }
 
-task jacocoMerge(type: JacocoMerge) {
+task jacocoReport(type: JacocoReport, group: 'verification') {
+    description = 'Generates an aggregate report from all subprojects'
     gradle.projectsEvaluated {
-        subprojects.each { 
-            jacocoMerge.dependsOn it.tasks.withType(JacocoReport)
-            jacocoMerge.executionData it.tasks.withType(JacocoReport).collect { it.executionData } 
+        subprojects.each {
+            jacocoReport.dependsOn it.tasks.withType(JacocoReport)
+            jacocoReport.executionData it.tasks.withType(JacocoReport).collect { it.executionData }
         }
     }
-    doFirst {
-        executionData = files(executionData.findAll { it.exists() })
-    }
-}
-
-task jacocoReport(type: JacocoReport, group: 'verification') {
-    description = 'Generates an aggregate report from all subprojects'
-    dependsOn jacocoMerge
-    executionData jacocoMerge.destinationFile
 
     reports {
-        html.enabled = true // human readable
-        xml.enabled = true
+        html.required = true // human readable
+        xml.required = true
     }
 
     gradle.projectsEvaluated {
diff --git a/build-tools/opensearchplugin-coverage.gradle b/build-tools/opensearchplugin-coverage.gradle
index adea8414e..df2c0513b 100644
--- a/build-tools/opensearchplugin-coverage.gradle
+++ b/build-tools/opensearchplugin-coverage.gradle
@@ -61,8 +61,8 @@ jacocoTestReport {
     getSourceDirectories().from(sourceSets.main.allSource)
     getClassDirectories().from(sourceSets.main.output)
     reports {
-        html.enabled = true // human readable
-        xml.enabled = true // for coverlay
+        html.required = true // human readable
+        xml.required = true // for coverlay
     }
 }
 
diff --git a/build-tools/pkgbuild.gradle b/build-tools/pkgbuild.gradle
index 4ac3eb325..8a70c13e3 100644
--- a/build-tools/pkgbuild.gradle
+++ b/build-tools/pkgbuild.gradle
@@ -1,4 +1,4 @@
-apply plugin: 'nebula.ospackage'
+apply plugin: 'com.netflix.nebula.ospackage'
 
 // This is afterEvaluate because the bundlePlugin ZIP task is updated afterEvaluate and changes the ZIP name to match the plugin name
 afterEvaluate {
@@ -8,7 +8,7 @@ afterEvaluate {
         version = "${project.version}" - "-SNAPSHOT"
 
         into '/usr/share/opensearch/plugins'
-        from(zipTree(bundlePlugin.archivePath)) {
+        from(zipTree(bundlePlugin.archiveFile)) {
             into opensearchplugin.name
         }
 
@@ -39,9 +39,8 @@ afterEvaluate {
         task renameRpm(type: Copy) {
             from("$buildDir/distributions")
             into("$buildDir/distributions")
-            include archiveName
-            rename archiveName, "${packageName}-${version}.rpm"
-            doLast { delete file("$buildDir/distributions/$archiveName") }
+            rename "$archiveFileName", "${packageName}-${archiveVersion}.rpm"
+            doLast { delete file("$buildDir/distributions/$archiveFileName") }
         }
     }
     
@@ -52,9 +51,8 @@ afterEvaluate {
         task renameDeb(type: Copy) {
             from("$buildDir/distributions")
             into("$buildDir/distributions")
-            include archiveName
-            rename archiveName, "${packageName}-${version}.deb"
-            doLast { delete file("$buildDir/distributions/$archiveName") }
+            rename "$archiveFileName", "${packageName}-${archiveVersion}.deb"
+            doLast { delete file("$buildDir/distributions/$archiveFileName") }
         }
     }
 }
\ No newline at end of file
diff --git a/build.gradle b/build.gradle
index 53d66e347..0cfe30ceb 100644
--- a/build.gradle
+++ b/build.gradle
@@ -7,7 +7,7 @@ buildscript {
     apply from: 'build-tools/repositories.gradle'
 
     ext {
-        opensearch_version = System.getProperty("opensearch.version", "2.11.0-SNAPSHOT")
+        opensearch_version = System.getProperty("opensearch.version", "2.12.0-SNAPSHOT")
         buildVersionQualifier = System.getProperty("build.version_qualifier", "")
         isSnapshot = "true" == System.getProperty("build.snapshot", "true")
         // 2.7.0-SNAPSHOT -> 2.7.0.0-SNAPSHOT
@@ -39,7 +39,7 @@ buildscript {
 }
 
 plugins {
-    id 'nebula.ospackage' version "8.3.0" apply false
+    id 'com.netflix.nebula.ospackage' version "11.5.0"
     id "com.dorongold.task-tree" version "1.5"
 }
 
@@ -48,7 +48,12 @@ apply plugin: 'jacoco'
 apply from: 'build-tools/merged-coverage.gradle'
 
 configurations {
-    ktlint
+    ktlint {
+      resolutionStrategy {
+          force "ch.qos.logback:logback-classic:1.3.14"
+          force "ch.qos.logback:logback-core:1.3.14"
+      }
+   }
 }
 
 dependencies {
diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/lock/LockModel.kt b/core/src/main/kotlin/org/opensearch/alerting/core/lock/LockModel.kt
new file mode 100644
index 000000000..706bd0987
--- /dev/null
+++ b/core/src/main/kotlin/org/opensearch/alerting/core/lock/LockModel.kt
@@ -0,0 +1,117 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.alerting.core.lock
+
+import org.opensearch.core.xcontent.ToXContent
+import org.opensearch.core.xcontent.ToXContentObject
+import org.opensearch.core.xcontent.XContentBuilder
+import org.opensearch.core.xcontent.XContentParser
+import org.opensearch.core.xcontent.XContentParserUtils
+import org.opensearch.index.seqno.SequenceNumbers
+import java.io.IOException
+import java.time.Instant
+
+class LockModel(
+    val lockId: String,
+    val scheduledJobId: String,
+    val lockTime: Instant,
+    val released: Boolean,
+    val seqNo: Long,
+    val primaryTerm: Long
+) : ToXContentObject {
+
+    constructor(
+        copyLock: LockModel,
+        seqNo: Long,
+        primaryTerm: Long
+    ) : this (
+        copyLock.lockId,
+        copyLock.scheduledJobId,
+        copyLock.lockTime,
+        copyLock.released,
+        seqNo,
+        primaryTerm
+    )
+
+    constructor(
+        copyLock: LockModel,
+        released: Boolean
+    ) : this (
+        copyLock.lockId,
+        copyLock.scheduledJobId,
+        copyLock.lockTime,
+        released,
+        copyLock.seqNo,
+        copyLock.primaryTerm
+    )
+
+    constructor(
+        copyLock: LockModel,
+        updateLockTime: Instant,
+        released: Boolean
+    ) : this (
+        copyLock.lockId,
+        copyLock.scheduledJobId,
+        updateLockTime,
+        released,
+        copyLock.seqNo,
+        copyLock.primaryTerm
+    )
+
+    constructor(
+        scheduledJobId: String,
+        lockTime: Instant,
+        released: Boolean
+    ) : this (
+        generateLockId(scheduledJobId),
+        scheduledJobId,
+        lockTime,
+        released,
+        SequenceNumbers.UNASSIGNED_SEQ_NO,
+        SequenceNumbers.UNASSIGNED_PRIMARY_TERM
+    )
+
+    override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder {
+        builder.startObject()
+            .field(SCHEDULED_JOB_ID, scheduledJobId)
+            .field(LOCK_TIME, lockTime.epochSecond)
+            .field(RELEASED, released)
+            .endObject()
+        return builder
+    }
+
+    companion object {
+        const val SCHEDULED_JOB_ID = "scheduled_job_id"
+        const val LOCK_TIME = "lock_time"
+        const val RELEASED = "released"
+
+        fun generateLockId(scheduledJobId: String): String {
+            return "$scheduledJobId-lock"
+        }
+
+        @JvmStatic
+        @JvmOverloads
+        @Throws(IOException::class)
+        fun parse(xcp: XContentParser, seqNo: Long, primaryTerm: Long): LockModel {
+            lateinit var scheduledJobId: String
+            lateinit var lockTime: Instant
+            var released: Boolean = false
+
+            XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp)
+            while (xcp.nextToken() != XContentParser.Token.END_OBJECT) {
+                val fieldName = xcp.currentName()
+                xcp.nextToken()
+
+                when (fieldName) {
+                    SCHEDULED_JOB_ID -> scheduledJobId = xcp.text()
+                    LOCK_TIME -> lockTime = Instant.ofEpochSecond(xcp.longValue())
+                    RELEASED -> released = xcp.booleanValue()
+                }
+            }
+            return LockModel(generateLockId(scheduledJobId), scheduledJobId, lockTime, released, seqNo, primaryTerm)
+        }
+    }
+}
diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/lock/LockService.kt b/core/src/main/kotlin/org/opensearch/alerting/core/lock/LockService.kt
new file mode 100644
index 000000000..35618e156
--- /dev/null
+++ b/core/src/main/kotlin/org/opensearch/alerting/core/lock/LockService.kt
@@ -0,0 +1,311 @@
+package org.opensearch.alerting.core.lock
+
+import org.apache.logging.log4j.LogManager
+import org.opensearch.ResourceAlreadyExistsException
+import org.opensearch.action.DocWriteResponse
+import org.opensearch.action.admin.indices.create.CreateIndexRequest
+import org.opensearch.action.admin.indices.create.CreateIndexResponse
+import org.opensearch.action.delete.DeleteRequest
+import org.opensearch.action.delete.DeleteResponse
+import org.opensearch.action.get.GetRequest
+import org.opensearch.action.get.GetResponse
+import org.opensearch.action.index.IndexRequest
+import org.opensearch.action.index.IndexResponse
+import org.opensearch.action.update.UpdateRequest
+import org.opensearch.action.update.UpdateResponse
+import org.opensearch.client.Client
+import org.opensearch.cluster.service.ClusterService
+import org.opensearch.common.settings.Settings
+import org.opensearch.common.xcontent.LoggingDeprecationHandler
+import org.opensearch.common.xcontent.XContentFactory
+import org.opensearch.common.xcontent.XContentType
+import org.opensearch.commons.alerting.model.ScheduledJob
+import org.opensearch.core.action.ActionListener
+import org.opensearch.core.xcontent.NamedXContentRegistry
+import org.opensearch.core.xcontent.ToXContent
+import org.opensearch.index.IndexNotFoundException
+import org.opensearch.index.engine.DocumentMissingException
+import org.opensearch.index.engine.VersionConflictEngineException
+import org.opensearch.index.seqno.SequenceNumbers
+import java.io.IOException
+import java.time.Instant
+
+private val log = LogManager.getLogger(LockService::class.java)
+
+class LockService(private val client: Client, private val clusterService: ClusterService) {
+    private var testInstant: Instant? = null
+
+    companion object {
+        const val LOCK_INDEX_NAME = ".opensearch-alerting-config-lock"
+
+        @JvmStatic
+        fun lockMapping(): String? {
+            return LockService::class.java.classLoader.getResource("mappings/opensearch-alerting-config-lock.json")
+                ?.readText()
+        }
+    }
+
+    fun lockIndexExist(): Boolean {
+        return clusterService.state().routingTable().hasIndex(LOCK_INDEX_NAME)
+    }
+
+    fun acquireLock(
+        scheduledJob: ScheduledJob,
+        listener: ActionListener<LockModel?>
+    ) {
+        val scheduledJobId = scheduledJob.id
+        acquireLockWithId(scheduledJobId, listener)
+    }
+
+    fun acquireLockWithId(
+        scheduledJobId: String,
+        listener: ActionListener<LockModel?>
+    ) {
+        val lockId = LockModel.generateLockId(scheduledJobId)
+        createLockIndex(
+            object : ActionListener<Boolean> {
+                override fun onResponse(created: Boolean) {
+                    if (created) {
+                        try {
+                            findLock(
+                                lockId,
+                                object : ActionListener<LockModel> {
+                                    override fun onResponse(existingLock: LockModel?) {
+                                        if (existingLock != null) {
+                                            if (isLockReleased(existingLock)) {
+                                                log.debug("lock is released or expired: {}", existingLock)
+                                                val updateLock = LockModel(existingLock, getNow(), false)
+                                                updateLock(updateLock, listener)
+                                            } else {
+                                                log.debug("Lock is NOT released or expired. {}", existingLock)
+                                                listener.onResponse(null)
+                                            }
+                                        } else {
+                                            val tempLock = LockModel(scheduledJobId, getNow(), false)
+                                            log.debug("Lock does not exist. Creating new lock {}", tempLock)
+                                            createLock(tempLock, listener)
+                                        }
+                                    }
+
+                                    override fun onFailure(e: Exception) {
+                                        listener.onFailure(e)
+                                    }
+                                }
+                            )
+                        } catch (e: VersionConflictEngineException) {
+                            log.debug("could not acquire lock {}", e.message)
+                            listener.onResponse(null)
+                        }
+                    } else {
+                        listener.onResponse(null)
+                    }
+                }
+
+                override fun onFailure(e: Exception) {
+                    listener.onFailure(e)
+                }
+            }
+        )
+    }
+
+    private fun createLock(
+        tempLock: LockModel,
+        listener: ActionListener<LockModel?>
+    ) {
+        try {
+            val request = IndexRequest(LOCK_INDEX_NAME).id(tempLock.lockId)
+                .source(tempLock.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS))
+                .setIfSeqNo(SequenceNumbers.UNASSIGNED_SEQ_NO)
+                .setIfPrimaryTerm(SequenceNumbers.UNASSIGNED_PRIMARY_TERM)
+                .create(true)
+            client.index(
+                request,
+                object : ActionListener<IndexResponse> {
+                    override fun onResponse(response: IndexResponse) {
+                        listener.onResponse(LockModel(tempLock, response.seqNo, response.primaryTerm))
+                    }
+
+                    override fun onFailure(e: Exception) {
+                        if (e is VersionConflictEngineException) {
+                            log.debug("Lock is already created. {}", e.message)
+                            listener.onResponse(null)
+                            return
+                        }
+                        listener.onFailure(e)
+                    }
+                }
+            )
+        } catch (ex: IOException) {
+            log.error("IOException occurred creating lock", ex)
+            listener.onFailure(ex)
+        }
+    }
+
+    private fun updateLock(
+        updateLock: LockModel,
+        listener: ActionListener<LockModel?>
+    ) {
+        try {
+            val updateRequest = UpdateRequest().index(LOCK_INDEX_NAME)
+                .id(updateLock.lockId)
+                .setIfSeqNo(updateLock.seqNo)
+                .setIfPrimaryTerm(updateLock.primaryTerm)
+                .doc(updateLock.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS))
+                .fetchSource(true)
+
+            client.update(
+                updateRequest,
+                object : ActionListener<UpdateResponse> {
+                    override fun onResponse(response: UpdateResponse) {
+                        listener.onResponse(LockModel(updateLock, response.seqNo, response.primaryTerm))
+                    }
+
+                    override fun onFailure(e: Exception) {
+                        if (e is VersionConflictEngineException) {
+                            log.debug("could not acquire lock {}", e.message)
+                        }
+                        if (e is DocumentMissingException) {
+                            log.debug(
+                                "Document is deleted. This happens if the job is already removed and" + " this is the last run." + "{}",
+                                e.message
+                            )
+                        }
+                        if (e is IOException) {
+                            log.error("IOException occurred updating lock.", e)
+                        }
+                        listener.onResponse(null)
+                    }
+                }
+            )
+        } catch (ex: IOException) {
+            log.error("IOException occurred updating lock.", ex)
+            listener.onResponse(null)
+        }
+    }
+
+    fun findLock(
+        lockId: String,
+        listener: ActionListener<LockModel>
+    ) {
+        val getRequest = GetRequest(LOCK_INDEX_NAME).id(lockId)
+        client.get(
+            getRequest,
+            object : ActionListener<GetResponse> {
+                override fun onResponse(response: GetResponse) {
+                    if (!response.isExists) {
+                        listener.onResponse(null)
+                    } else {
+                        try {
+                            val parser = XContentType.JSON.xContent()
+                                .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, response.sourceAsString)
+                            parser.nextToken()
+                            listener.onResponse(LockModel.parse(parser, response.seqNo, response.primaryTerm))
+                        } catch (e: IOException) {
+                            log.error("IOException occurred finding lock", e)
+                            listener.onResponse(null)
+                        }
+                    }
+                }
+
+                override fun onFailure(e: Exception) {
+                    log.error("Exception occurred finding lock", e)
+                    listener.onFailure(e)
+                }
+            }
+        )
+    }
+
+    fun release(
+        lock: LockModel?,
+        listener: ActionListener<Boolean>
+    ) {
+        if (lock == null) {
+            log.debug("Lock is null. Nothing to release.")
+            listener.onResponse(false)
+        } else {
+            log.debug("Releasing lock: {}", lock)
+            val lockToRelease = LockModel(lock, true)
+            updateLock(
+                lockToRelease,
+                object : ActionListener<LockModel?> {
+                    override fun onResponse(releasedLock: LockModel?) {
+                        listener.onResponse(releasedLock != null)
+                    }
+
+                    override fun onFailure(e: Exception) {
+                        listener.onFailure(e)
+                    }
+                }
+            )
+        }
+    }
+
+    fun deleteLock(
+        lockId: String,
+        listener: ActionListener<Boolean>
+    ) {
+        val deleteRequest = DeleteRequest(LOCK_INDEX_NAME).id(lockId)
+        client.delete(
+            deleteRequest,
+            object : ActionListener<DeleteResponse> {
+                override fun onResponse(response: DeleteResponse) {
+                    listener.onResponse(
+                        response.result == DocWriteResponse.Result.DELETED || response.result == DocWriteResponse.Result.NOT_FOUND
+                    )
+                }
+
+                override fun onFailure(e: Exception) {
+                    if (e is IndexNotFoundException || e.cause is IndexNotFoundException) {
+                        log.debug("Index is not found to delete lock. {}", e.message)
+                        listener.onResponse(true)
+                    } else {
+                        listener.onFailure(e)
+                    }
+                }
+            }
+        )
+    }
+
+    private fun createLockIndex(listener: ActionListener<Boolean>) {
+        if (lockIndexExist()) {
+            listener.onResponse(true)
+        } else {
+            val indexRequest = CreateIndexRequest(LOCK_INDEX_NAME).mapping(lockMapping())
+                .settings(Settings.builder().put("index.hidden", true).build())
+            client.admin().indices().create(
+                indexRequest,
+                object : ActionListener<CreateIndexResponse> {
+                    override fun onResponse(response: CreateIndexResponse) {
+                        listener.onResponse(response.isAcknowledged)
+                    }
+
+                    override fun onFailure(ex: Exception) {
+                        log.error("Failed to update config index schema", ex)
+                        if (ex is ResourceAlreadyExistsException || ex.cause is ResourceAlreadyExistsException
+                        ) {
+                            listener.onResponse(true)
+                        } else {
+                            listener.onFailure(ex)
+                        }
+                    }
+                }
+            )
+        }
+    }
+
+    private fun isLockReleased(lock: LockModel): Boolean {
+        return lock.released
+    }
+
+    private fun getNow(): Instant {
+        return if (testInstant != null) {
+            testInstant!!
+        } else {
+            Instant.now()
+        }
+    }
+
+    fun setTime(testInstant: Instant) {
+        this.testInstant = testInstant
+    }
+}
diff --git a/core/src/main/resources/mappings/opensearch-alerting-config-lock.json b/core/src/main/resources/mappings/opensearch-alerting-config-lock.json
new file mode 100644
index 000000000..401374a8f
--- /dev/null
+++ b/core/src/main/resources/mappings/opensearch-alerting-config-lock.json
@@ -0,0 +1,18 @@
+{
+  "dynamic": "strict",
+  "properties": {
+    "scheduled_job_id": {
+      "type": "keyword"
+    },
+    "lock_time": {
+      "type": "date",
+      "format": "epoch_second"
+    },
+    "lock_duration_seconds": {
+      "type": "long"
+    },
+    "released": {
+      "type": "boolean"
+    }
+  }
+}
\ No newline at end of file
diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar
index 943f0cbfa..7f93135c4 100644
Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ
diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
index 774fae876..3499ded5c 100644
--- a/gradle/wrapper/gradle-wrapper.properties
+++ b/gradle/wrapper/gradle-wrapper.properties
@@ -1,5 +1,6 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.6.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip
+networkTimeout=10000
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
diff --git a/gradlew b/gradlew
index 65dcd68d6..1aa94a426 100755
--- a/gradlew
+++ b/gradlew
@@ -83,10 +83,8 @@ done
 # This is normally unused
 # shellcheck disable=SC2034
 APP_BASE_NAME=${0##*/}
-APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit
-
-# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
+# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
+APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit
 
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD=maximum
@@ -133,10 +131,13 @@ location of your Java installation."
     fi
 else
     JAVACMD=java
-    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+    if ! command -v java >/dev/null 2>&1
+    then
+        die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
 
 Please set the JAVA_HOME variable in your environment to match the
 location of your Java installation."
+    fi
 fi
 
 # Increase the maximum file descriptors if we can.
@@ -144,7 +145,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
     case $MAX_FD in #(
       max*)
         # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
-        # shellcheck disable=SC3045 
+        # shellcheck disable=SC2039,SC3045
         MAX_FD=$( ulimit -H -n ) ||
             warn "Could not query maximum file descriptor limit"
     esac
@@ -152,7 +153,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
       '' | soft) :;; #(
       *)
         # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
-        # shellcheck disable=SC3045 
+        # shellcheck disable=SC2039,SC3045
         ulimit -n "$MAX_FD" ||
             warn "Could not set maximum file descriptor limit to $MAX_FD"
     esac
@@ -197,11 +198,15 @@ if "$cygwin" || "$msys" ; then
     done
 fi
 
-# Collect all arguments for the java command;
-#   * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of
-#     shell script including quotes and variable substitutions, so put them in
-#     double quotes to make sure that they get re-expanded; and
-#   * put everything else in single quotes, so that it's not re-expanded.
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
+
+# Collect all arguments for the java command:
+#   * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
+#     and any embedded shellness will be escaped.
+#   * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
+#     treated as '${Hostname}' itself on the command line.
 
 set -- \
         "-Dorg.gradle.appname=$APP_BASE_NAME" \
diff --git a/release-notes/opensearch-alerting.release-notes-2.11.1.0.md b/release-notes/opensearch-alerting.release-notes-2.11.1.0.md
new file mode 100644
index 000000000..409559669
--- /dev/null
+++ b/release-notes/opensearch-alerting.release-notes-2.11.1.0.md
@@ -0,0 +1,15 @@
+## Version 2.11.1.0 2023-11-21
+Compatible with OpenSearch 2.11.1
+
+### Maintenance
+* Increment version to 2.11.1-SNAPSHOT. ([#1274](https://github.com/opensearch-project/alerting/pull/1274))
+
+### Bug Fixes
+* Fix for ConcurrentModificationException with linkedHashmap. ([#1255](https://github.com/opensearch-project/alerting/pull/1255))
+
+### Refactoring
+* Handle Doc level query 'fields' param for query string query. ([#1240](https://github.com/opensearch-project/alerting/pull/1240))
+* Add nested fields param mapping findings index for doc level queries. ([#1276](https://github.com/opensearch-project/alerting/pull/1276))
+
+### Documentation
+* Added 2.11.1 release notes. ([#1306](https://github.com/opensearch-project/alerting/pull/1306))
\ No newline at end of file
diff --git a/release-notes/opensearch-alerting.release-notes-2.12.0.0.md b/release-notes/opensearch-alerting.release-notes-2.12.0.0.md
new file mode 100644
index 000000000..1ecdeadd7
--- /dev/null
+++ b/release-notes/opensearch-alerting.release-notes-2.12.0.0.md
@@ -0,0 +1,27 @@
+## Version 2.12.0.0 2024-02-06
+Compatible with OpenSearch 2.12.0
+
+### Maintenance
+* Increment version to 2.12.0-SNAPSHOT. ([#1239](https://github.com/opensearch-project/alerting/pull/1239))
+* Removed default admin credentials for alerting ([#1399](https://github.com/opensearch-project/alerting/pull/1399))
+* ipaddress lib upgrade as part of cve fix ([#1397](https://github.com/opensearch-project/alerting/pull/1397))
+
+### Bug Fixes
+* Don't attempt to parse workflow if it doesn't exist ([#1346](https://github.com/opensearch-project/alerting/pull/1346))
+* Set docData to empty string if actual is null ([#1325](https://github.com/opensearch-project/alerting/pull/1325))
+
+### Enhancements
+* Optimize doc-level monitor execution workflow for datastreams ([#1302](https://github.com/opensearch-project/alerting/pull/1302))
+* Inject namedWriteableRegistry during ser/deser of SearchMonitorAction ([#1382](https://github.com/opensearch-project/alerting/pull/1382))
+* Bulk index findings and sequentially invoke auto-correlations ([#1355](https://github.com/opensearch-project/alerting/pull/1355))
+* Implemented cross-cluster monitor support ([#1404](https://github.com/opensearch-project/alerting/pull/1404))
+
+### Refactoring
+* Reference get monitor and search monitor action / request / responses from common-utils ([#1315](https://github.com/opensearch-project/alerting/pull/1315))
+
+### Infrastructure
+* Fix workflow security tests. ([#1310](https://github.com/opensearch-project/alerting/pull/1310))
+* Upgrade to Gradle 8.5 ([#1369](https://github.com/opensearch-project/alerting/pull/1369))
+
+### Documentation
+* Added 2.12 release notes ([#1408](https://github.com/opensearch-project/alerting/pull/1408))
\ No newline at end of file
diff --git a/scripts/build.sh b/scripts/build.sh
index 5a173adfb..9f0afc3da 100755
--- a/scripts/build.sh
+++ b/scripts/build.sh
@@ -74,6 +74,7 @@ distributions="$(dirname "${zipPath}")"
 echo "COPY ${distributions}/*.zip"
 cp ${distributions}/*.zip ./$OUTPUT/plugins
 
+./gradlew publishToMavenLocal -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER
 ./gradlew publishPluginZipPublicationToZipStagingRepository -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER
 mkdir -p $OUTPUT/maven/org/opensearch
 cp -r ./build/local-staging-repo/org/opensearch/. $OUTPUT/maven/org/opensearch