Retry assertNoLeaks multiple times.

ratis-common/src/main/java/org/apache/ratis/util/LeakDetector.java

@@ -79,9 +79,9 @@ synchronized LeakTracker add(Object referent, ReferenceQueue<Object> queue, Supp
       return tracker;
     }
 
-    synchronized void assertNoLeaks() {
+    synchronized int getNumLeaks(boolean throwException) {
       if (set.isEmpty()) {
-        return;
+        return 0;
       }
 
       int n = 0;
@@ -90,7 +90,10 @@ synchronized void assertNoLeaks() {
           n++;
         }
       }
-      assertNoLeaks(n);
+      if (throwException) {
+        assertNoLeaks(n);
+      }
+      return n;
     }
 
     synchronized void assertNoLeaks(int leaks) {
@@ -151,12 +154,21 @@ Runnable track(Object leakable, Supplier<String> reportLeak) {
     return allLeaks.add(leakable, queue, reportLeak)::remove;
   }
 
-  public void assertNoLeaks() {
+  public void assertNoLeaks(int maxRetries) throws InterruptedException {
     synchronized (leakMessages) {
       Preconditions.assertTrue(leakMessages.isEmpty(),
           () -> "#leaks = " + leakMessages.size() + "\n" + leakMessages);
     }
-    allLeaks.assertNoLeaks();
+
+    for(int i = 0; i < maxRetries; i++) {
+      final int numLeaks = allLeaks.getNumLeaks(false);
+      if (numLeaks == 0) {
+        return;
+      }
+      LOG.warn("{}/{}) numLeaks == {} > 0, will wait and retry ...", i, maxRetries, numLeaks);
+      TimeDuration.ONE_SECOND.sleep();
+    }
+    allLeaks.getNumLeaks(true);
   }
 
   private static final class LeakTracker extends WeakReference<Object> {

ratis-server/src/test/java/org/apache/ratis/server/impl/MiniRaftCluster.java

@@ -854,6 +854,7 @@ public void shutdown() {
     LOG.info("*** ");
     LOG.info("************************************************************** ");
     LOG.info(printServers());
+    final int maxRetries = 30;
 
     // TODO: classes like RaftLog may throw uncaught exception during shutdown (e.g. write after close)
     ExitUtils.setTerminateOnUncaughtException(false);
@@ -864,7 +865,19 @@ public void shutdown() {
     try {
       executor.shutdown();
       // just wait for a few seconds
-      executor.awaitTermination(5, TimeUnit.SECONDS);
+      boolean terminated = false;
+
+      for(int i = 0; i < maxRetries && !terminated; ) {
+        terminated = executor.awaitTermination(1, TimeUnit.SECONDS);
+        if (!terminated) {
+          i++;
+          if (i < maxRetries) {
+            LOG.warn("Not yet able to shutdown executor {}/{}, will wait again ...", i, maxRetries);
+          } else {
+            LOG.error("Failed to shutdown executor, some servers may be still running:\n{}", printServers());
+          }
+        }
+      }
     } catch (InterruptedException e) {
       LOG.warn("shutdown interrupted", e);
       Thread.currentThread().interrupt();
@@ -878,9 +891,13 @@ public void shutdown() {
     try {
       RaftTestUtil.gc();
     } catch (InterruptedException e) {
-      LOG.info("gc interrupted.");
+      LOG.warn("gc interrupted.", e);
+    }
+    try {
+      ReferenceCountedLeakDetector.getLeakDetector().assertNoLeaks(maxRetries);
+    } catch (InterruptedException e) {
+      LOG.warn("LeakDetector interrupted.", e);
     }
-    ReferenceCountedLeakDetector.getLeakDetector().assertNoLeaks();
   }
 
   /**