From 6239d7b665a98efcee7a1001e12a215d9ed53be4 Mon Sep 17 00:00:00 2001 From: Subhobrata Dey Date: Wed, 2 Oct 2024 16:29:46 -0700 Subject: [PATCH] delete query index only if put mappings throws an exception (#1685) Signed-off-by: Subhobrata Dey --- .../alerting/DocumentLevelMonitorRunner.kt | 11 ---- .../alerting/util/DocLevelMonitorQueries.kt | 51 +++++++++++++++++-- .../SampleRemoteMonitorRestHandler.java | 6 +-- 3 files changed, 49 insertions(+), 19 deletions(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/DocumentLevelMonitorRunner.kt b/alerting/src/main/kotlin/org/opensearch/alerting/DocumentLevelMonitorRunner.kt index 6f339fdce..b5b2c4f49 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/DocumentLevelMonitorRunner.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/DocumentLevelMonitorRunner.kt @@ -121,17 +121,6 @@ class DocumentLevelMonitorRunner : MonitorRunner() { throw IndexNotFoundException(docLevelMonitorInput.indices.joinToString(",")) } - if (monitor.deleteQueryIndexInEveryRun == true && - monitorCtx.docLevelMonitorQueries!!.docLevelQueryIndexExists(monitor.dataSources) - ) { - val ack = monitorCtx.docLevelMonitorQueries!!.deleteDocLevelQueryIndex(monitor.dataSources) - if (!ack) { - logger.error( - "Deletion of concrete queryIndex:${monitor.dataSources.queryIndex} is not ack'd! " + - "for monitor ${monitor.id}" - ) - } - } monitorCtx.docLevelMonitorQueries!!.initDocLevelQueryIndex(monitor.dataSources) monitorCtx.docLevelMonitorQueries!!.indexDocLevelQueries( monitor = monitor, diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/DocLevelMonitorQueries.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/DocLevelMonitorQueries.kt index 7ce3309db..d47537c7f 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/DocLevelMonitorQueries.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/DocLevelMonitorQueries.kt @@ -358,7 +358,8 @@ class DocLevelMonitorQueries(private val client: Client, private val clusterServ monitorMetadata, updatedIndexName, sourceIndexFieldLimit, - updatedProperties + updatedProperties, + indexTimeout ) if (updateMappingResponse.isAcknowledged) { @@ -487,7 +488,8 @@ class DocLevelMonitorQueries(private val client: Client, private val clusterServ monitorMetadata: MonitorMetadata, sourceIndex: String, sourceIndexFieldLimit: Long, - updatedProperties: MutableMap + updatedProperties: MutableMap, + indexTimeout: TimeValue ): Pair { var targetQueryIndex = monitorMetadata.sourceToQueryIndexMapping[sourceIndex + monitor.id] if ( @@ -551,9 +553,48 @@ class DocLevelMonitorQueries(private val client: Client, private val clusterServ } } } else { - log.debug("unknown exception during PUT mapping on queryIndex: $targetQueryIndex") - val unwrappedException = ExceptionsHelper.unwrapCause(e) as Exception - throw AlertingException.wrap(unwrappedException) + // retry with deleting query index + if (monitor.deleteQueryIndexInEveryRun == true) { + try { + log.error( + "unknown exception during PUT mapping on queryIndex: $targetQueryIndex, " + + "retrying with deletion of query index", + e + ) + if (docLevelQueryIndexExists(monitor.dataSources)) { + val ack = monitorCtx.docLevelMonitorQueries!!.deleteDocLevelQueryIndex(monitor.dataSources) + if (!ack) { + log.error( + "Deletion of concrete queryIndex:${monitor.dataSources.queryIndex} is not ack'd! " + + "for monitor ${monitor.id}" + ) + } + } + initDocLevelQueryIndex(monitor.dataSources) + indexDocLevelQueries( + monitor = monitor, + monitorId = monitor.id, + monitorMetadata, + indexTimeout = indexTimeout + ) + } catch (e: Exception) { + log.error( + "Doc level monitor ${monitor.id}: unknown exception during " + + "PUT mapping on queryIndex: $targetQueryIndex", + e + ) + val unwrappedException = ExceptionsHelper.unwrapCause(e) as Exception + throw AlertingException.wrap(unwrappedException) + } + } else { + log.error( + "Doc level monitor ${monitor.id}: unknown exception during " + + "PUT mapping on queryIndex: $targetQueryIndex", + e + ) + val unwrappedException = ExceptionsHelper.unwrapCause(e) as Exception + throw AlertingException.wrap(unwrappedException) + } } } // We did rollover, so try to apply mappings again on new targetQueryIndex diff --git a/sample-remote-monitor-plugin/src/main/java/org/opensearch/alerting/SampleRemoteMonitorRestHandler.java b/sample-remote-monitor-plugin/src/main/java/org/opensearch/alerting/SampleRemoteMonitorRestHandler.java index a8f384cb6..9875ef55f 100644 --- a/sample-remote-monitor-plugin/src/main/java/org/opensearch/alerting/SampleRemoteMonitorRestHandler.java +++ b/sample-remote-monitor-plugin/src/main/java/org/opensearch/alerting/SampleRemoteMonitorRestHandler.java @@ -94,7 +94,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient "id", null)), trigger1Serialized)), Map.of(), new DataSources(), - true, + false, "sample-remote-monitor-plugin" ); IndexMonitorRequest indexMonitorRequest1 = new IndexMonitorRequest( @@ -155,7 +155,7 @@ public void onFailure(Exception e) { List.of(), Map.of(), new DataSources(), - true, + false, "sample-remote-monitor-plugin" ); IndexMonitorRequest indexMonitorRequest2 = new IndexMonitorRequest( @@ -239,7 +239,7 @@ public void onFailure(Exception e) { "id", null)), trigger1Serialized)), Map.of(), new DataSources(), - true, + false, "sample-remote-monitor-plugin" ); IndexMonitorRequest indexDocLevelMonitorRequest = new IndexMonitorRequest(