Handle monitor or monitor index not exists during detector deletion

build.gradle

@@ -6,7 +6,7 @@ import org.opensearch.gradle.test.RestIntegTestTask
 
 buildscript {
     ext {
-        opensearch_version = System.getProperty("opensearch.version", "2.5.0-SNAPSHOT")
+        opensearch_version = System.getProperty("opensearch.version", "2.6.0-SNAPSHOT")
         isSnapshot = "true" == System.getProperty("build.snapshot", "true")
         buildVersionQualifier = System.getProperty("build.version_qualifier", "")
         version_tokens = opensearch_version.tokenize('-')

src/main/java/org/opensearch/securityanalytics/transport/TransportDeleteDetectorAction.java

@@ -11,7 +11,6 @@
 import java.util.concurrent.atomic.AtomicReference;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.apache.lucene.util.SetOnce;
 import org.opensearch.OpenSearchStatusException;
 import org.opensearch.action.ActionListener;
 import org.opensearch.action.ActionRunnable;
@@ -25,6 +24,7 @@
 import org.opensearch.action.support.WriteRequest;
 import org.opensearch.client.Client;
 import org.opensearch.client.node.NodeClient;
+import org.opensearch.common.SetOnce;
 import org.opensearch.common.inject.Inject;
 import org.opensearch.common.xcontent.LoggingDeprecationHandler;
 import org.opensearch.common.xcontent.NamedXContentRegistry;
@@ -138,7 +138,6 @@ public void onFailure(Exception t) {
 
         private void onGetResponse(Detector detector) {
             List<String> monitorIds = detector.getMonitorIds();
-            String ruleIndex = detector.getRuleIndex();
             ActionListener<DeleteMonitorResponse> deletesListener = new GroupedActionListener<>(new ActionListener<>() {
                 @Override
                 public void onResponse(Collection<DeleteMonitorResponse> responses) {
@@ -158,8 +157,13 @@ public void onResponse(Collection<DeleteMonitorResponse> responses) {
 
                 @Override
                 public void onFailure(Exception e) {
-                    if (counter.compareAndSet(false, true)) {
-                        finishHim(null, e);
+                    if(isOnlyMonitorOrIndexMissingExceptionThrownByGroupedActionListener(e, detector.getId())) {
+                        deleteDetectorFromConfig(detector.getId(), request.getRefreshPolicy());
+                    } else {
+                        log.error(String.format(Locale.ROOT, "Failed to delete detector %s", detector.getId()), e);
+                        if (counter.compareAndSet(false, true)) {
+                            finishHim(null, e);
+                        }
                     }
                 }
             }, monitorIds.size());
@@ -212,6 +216,7 @@ private void onFailures(Exception t) {
         private void finishHim(String detectorId, Exception t) {
             threadPool.executor(ThreadPool.Names.GENERIC).execute(ActionRunnable.supply(listener, () -> {
                 if (t != null) {
+                    log.error(String.format(Locale.ROOT, "Failed to delete detector %s",detectorId), t);
                     if (t instanceof OpenSearchStatusException) {
                         throw t;
                     }
@@ -221,5 +226,29 @@ private void finishHim(String detectorId, Exception t) {
                 }
             }));
         }
+
+        private boolean isOnlyMonitorOrIndexMissingExceptionThrownByGroupedActionListener(
+                Exception ex,
+                String detectorId
+        ) {
+            // grouped action listener listens on mutliple listeners but throws only one exception. If multiple
+            // listeners fail the other exceptions are added as suppressed exceptions to the first failure.
+            int len = ex.getSuppressed().length;
+            for (int i = 0; i <= len; i++) {
+                Throwable e = i == len ? ex : ex.getSuppressed()[i];
+                if (e.getMessage().matches("(.*)Monitor(.*) is not found(.*)")
+                                || e.getMessage().contains(
+                                        "Configured indices are not found: [.opendistro-alerting-config]")
+                ) {
+                    log.error(
+                            String.format(Locale.ROOT, "Monitor or jobs index already deleted." +
+                                    " Proceeding with detector %s deletion", detectorId),
+                            e);
+                } else {
+                    return false;
+                }
+            }
+            return true;
+        }
     }
 }

src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java

@@ -8,7 +8,6 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.search.join.ScoreMode;
-import org.apache.lucene.util.SetOnce;
 import org.opensearch.OpenSearchStatusException;
 import org.opensearch.action.ActionListener;
 import org.opensearch.action.ActionRunnable;
@@ -32,6 +31,7 @@
 import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
 import org.opensearch.cluster.metadata.MappingMetadata;
 import org.opensearch.cluster.service.ClusterService;
+import org.opensearch.common.SetOnce;
 import org.opensearch.common.inject.Inject;
 import org.opensearch.common.io.stream.NamedWriteableRegistry;
 import org.opensearch.common.settings.Settings;

src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java

@@ -238,6 +238,18 @@ protected Response executeAlertingMonitor(RestClient client, String monitorId, M
         return makeRequest(client, "POST", String.format(Locale.getDefault(), "/_plugins/_alerting/monitors/%s/_execute", monitorId), params, null);
     }
 
+    protected Response deleteAlertingMonitorIndex() throws IOException {
+        return makeRequest(client(), "DELETE", String.format(Locale.getDefault(), "/.opendistro-alerting-config"), new HashMap<>(), null);
+    }
+
+    protected Response deleteAlertingMonitor(String monitorId) throws IOException {
+        return deleteAlertingMonitor(client(), monitorId);
+    }
+
+    protected Response deleteAlertingMonitor(RestClient client, String monitorId) throws IOException {
+        return makeRequest(client, "DELETE", String.format(Locale.getDefault(), "/_plugins/_alerting/monitors/%s", monitorId), new HashMap<>(), null);
+    }
+
     protected List<SearchHit> executeSearch(String index, String request) throws IOException {
         return executeSearch(index, request, true);
     }

src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java

@@ -710,6 +710,48 @@ public void testDeletingADetector_single_ruleTopicIndex() throws IOException {
         Assert.assertEquals(0, hits.size());
     }
 
+    @SuppressWarnings("unchecked")
+    public void testDeletingADetector_MonitorNotExists() throws IOException {
+        String index = createTestIndex(randomIndex(), windowsIndexMapping());
+
+        // Execute CreateMappingsAction to add alias mapping for index
+        Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI);
+        // both req params and req body are supported
+        createMappingRequest.setJsonEntity(
+                "{ \"index_name\":\"" + index + "\"," +
+                        "  \"rule_topic\":\"" + randomDetectorType() + "\", " +
+                        "  \"partial\":true" +
+                        "}"
+        );
+
+        Response response = client().performRequest(createMappingRequest);
+        assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode());
+        // Create detector #1 of type test_windows
+        Detector detector1 = randomDetectorWithTriggers(getRandomPrePackagedRules(), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of())));
+        String detectorId1 = createDetector(detector1);
+
+        String request = "{\n" +
+                "   \"query\" : {\n" +
+                "     \"match\":{\n" +
+                "        \"_id\": \"" + detectorId1 + "\"\n" +
+                "     }\n" +
+                "   }\n" +
+                "}";
+        List<SearchHit> hits = executeSearch(Detector.DETECTORS_INDEX, request);
+        SearchHit hit = hits.get(0);
+
+        String monitorId = ((List<String>) ((Map<String, Object>) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0);
+
+        Response deleteMonitorResponse = deleteAlertingMonitor(monitorId);
+        assertEquals(200, deleteMonitorResponse.getStatusLine().getStatusCode());
+        entityAsMap(deleteMonitorResponse);
+
+        Response deleteResponse = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId1, Collections.emptyMap(), null);
+        Assert.assertEquals("Delete detector failed", RestStatus.OK, restStatus(deleteResponse));
+        hits = executeSearch(Detector.DETECTORS_INDEX, request);
+        Assert.assertEquals(0, hits.size());
+    }
+
     public void testDeletingADetector_oneDetectorType_multiple_ruleTopicIndex() throws IOException {
         String index1 = "test_index_1";
         createIndex(index1, Settings.EMPTY);